From e87ef036056b58d576ffe61ddfa98732f17cb184 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 5 May 2023 22:08:19 +0100 Subject: [PATCH 001/152] Pass the `LATEST_SALT_RELEASE` environment variables through to the VM Signed-off-by: Pedro Algarvio --- .github/workflows/test-package-downloads-action-linux.yml | 4 ++-- .github/workflows/test-package-downloads-action-windows.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-package-downloads-action-linux.yml b/.github/workflows/test-package-downloads-action-linux.yml index ee67c4d4020..7df9ec1c8f3 100644 --- a/.github/workflows/test-package-downloads-action-linux.yml +++ b/.github/workflows/test-package-downloads-action-linux.yml @@ -224,7 +224,7 @@ jobs: run: | tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \ -E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING \ + -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE \ --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }} -- download-pkgs - name: Run Package Download Tests @@ -241,7 +241,7 @@ jobs: run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ -E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING \ + -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE \ --nox-session=${{ inputs.nox-session }} --rerun-failures ${{ inputs.distro-slug }} -- download-pkgs - name: Combine Coverage Reports diff --git a/.github/workflows/test-package-downloads-action-windows.yml b/.github/workflows/test-package-downloads-action-windows.yml index 10d4462e451..963372925d2 100644 --- a/.github/workflows/test-package-downloads-action-windows.yml +++ b/.github/workflows/test-package-downloads-action-windows.yml @@ -234,7 +234,7 @@ jobs: run: | tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \ -E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING \ + -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE \ --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }} -- download-pkgs - name: Run Package Download Tests @@ -252,7 +252,7 @@ jobs: run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ -E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING \ + -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE \ --nox-session=${{ inputs.nox-session }} --rerun-failures ${{ inputs.distro-slug }} -- download-pkgs - name: Combine Coverage Reports From 3cd21ceb88c0a45700e157f8cceb1f53ac2ab4e2 Mon Sep 17 00:00:00 2001 From: MKLeb Date: Tue, 2 May 2023 20:53:24 -0400 Subject: [PATCH 002/152] Refactor the `tools pkg repo` commands into a subdirectory --- tools/__init__.py | 2 + tools/pkg/repo.py | 1906 ------------------------------------ tools/pkg/repo/__init__.py | 181 ++++ tools/pkg/repo/create.py | 1038 ++++++++++++++++++++ tools/pkg/repo/publish.py | 653 ++++++++++++ tools/utils.py | 127 +++ 6 files changed, 2001 insertions(+), 1906 deletions(-) delete mode 100644 tools/pkg/repo.py create mode 100644 tools/pkg/repo/__init__.py create mode 100644 tools/pkg/repo/create.py create mode 100644 tools/pkg/repo/publish.py diff --git a/tools/__init__.py b/tools/__init__.py index 419ec309c2f..02e6b8de903 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -8,6 +8,8 @@ ptscripts.register_tools_module("tools.docs") ptscripts.register_tools_module("tools.pkg") ptscripts.register_tools_module("tools.pkg.repo") ptscripts.register_tools_module("tools.pkg.build") +ptscripts.register_tools_module("tools.pkg.repo.create") +ptscripts.register_tools_module("tools.pkg.repo.publish") ptscripts.register_tools_module("tools.pre_commit") ptscripts.register_tools_module("tools.release") ptscripts.register_tools_module("tools.vm") diff --git a/tools/pkg/repo.py b/tools/pkg/repo.py deleted file mode 100644 index d781cf3c8ff..00000000000 --- a/tools/pkg/repo.py +++ /dev/null @@ -1,1906 +0,0 @@ -""" -These commands are used to build the pacakge repository files. -""" -# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated -from __future__ import annotations - -import fnmatch -import hashlib -import json -import logging -import os -import pathlib -import re -import shutil -import sys -import tempfile -import textwrap -from datetime import datetime -from typing import TYPE_CHECKING, Any - -import packaging.version -from ptscripts import Context, command_group - -import tools.pkg -import tools.utils -from tools.utils import Version, get_salt_releases - -try: - import boto3 - from botocore.exceptions import ClientError -except ImportError: - print( - "\nPlease run 'python -m pip install -r " - "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info), - file=sys.stderr, - flush=True, - ) - raise - -log = logging.getLogger(__name__) - -# Define the command group -repo = command_group( - name="repo", - help="Packaging Repository Related Commands", - description=__doc__, - parent="pkg", -) - -create = command_group( - name="create", help="Packaging Repository Creation Related Commands", parent=repo -) - -publish = command_group( - name="publish", - help="Packaging Repository Publication Related Commands", - parent=repo, -) - - -_deb_distro_info = { - "debian": { - "10": { - "label": "deb10ary", - "codename": "buster", - "suitename": "oldstable", - }, - "11": { - "label": "deb11ary", - "codename": "bullseye", - "suitename": "stable", - }, - }, - "ubuntu": { - "20.04": { - "label": "salt_ubuntu2004", - "codename": "focal", - }, - "22.04": { - "label": "salt_ubuntu2204", - "codename": "jammy", - }, - }, -} - - -@create.command( - name="deb", - arguments={ - "salt_version": { - "help": ( - "The salt version for which to build the repository configuration files. " - "If not passed, it will be discovered by running 'python3 salt/version.py'." - ), - "required": True, - }, - "distro": { - "help": "The debian based distribution to build the repository for", - "choices": list(_deb_distro_info), - "required": True, - }, - "distro_version": { - "help": "The distro version.", - "required": True, - }, - "distro_arch": { - "help": "The distribution architecture", - "choices": ("x86_64", "amd64", "aarch64", "arm64"), - }, - "repo_path": { - "help": "Path where the repository shall be created.", - "required": True, - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "incoming": { - "help": ( - "The path to the directory containing the files that should added to " - "the repository." - ), - "required": True, - }, - "nightly_build_from": { - "help": "Developement repository target", - }, - }, -) -def debian( - ctx: Context, - salt_version: str = None, - distro: str = None, - distro_version: str = None, - incoming: pathlib.Path = None, - repo_path: pathlib.Path = None, - key_id: str = None, - distro_arch: str = "amd64", - nightly_build_from: str = None, -): - """ - Create the debian repository. - """ - if TYPE_CHECKING: - assert salt_version is not None - assert distro is not None - assert distro_version is not None - assert incoming is not None - assert repo_path is not None - assert key_id is not None - display_name = f"{distro.capitalize()} {distro_version}" - if distro_version not in _deb_distro_info[distro]: - ctx.error(f"Support for {display_name} is missing.") - ctx.exit(1) - - if distro_arch == "x86_64": - ctx.info(f"The {distro_arch} arch is an alias for 'amd64'. Adjusting.") - distro_arch = "amd64" - - if distro_arch == "aarch64": - ctx.info(f"The {distro_arch} arch is an alias for 'arm64'. Adjusting.") - distro_arch = "arm64" - - distro_details = _deb_distro_info[distro][distro_version] - - ctx.info("Distribution Details:") - ctx.info(distro_details) - if TYPE_CHECKING: - assert isinstance(distro_details["label"], str) - assert isinstance(distro_details["codename"], str) - assert isinstance(distro_details["suitename"], str) - label: str = distro_details["label"] - codename: str = distro_details["codename"] - - ftp_archive_config_suite = "" - if distro == "debian": - suitename: str = distro_details["suitename"] - ftp_archive_config_suite = ( - f"""\n APT::FTPArchive::Release::Suite "{suitename}";\n""" - ) - archive_description = f"SaltProject {display_name} Python 3{'' if not nightly_build_from else ' development'} Salt package repo" - ftp_archive_config = f"""\ - APT::FTPArchive::Release::Origin "SaltProject"; - APT::FTPArchive::Release::Label "{label}";{ftp_archive_config_suite} - APT::FTPArchive::Release::Codename "{codename}"; - APT::FTPArchive::Release::Architectures "{distro_arch}"; - APT::FTPArchive::Release::Components "main"; - APT::FTPArchive::Release::Description "{archive_description}"; - APT::FTPArchive::Release::Acquire-By-Hash "yes"; - Dir {{ - ArchiveDir "."; - }}; - BinDirectory "pool" {{ - Packages "dists/{codename}/main/binary-{distro_arch}/Packages"; - Sources "dists/{codename}/main/source/Sources"; - Contents "dists/{codename}/main/Contents-{distro_arch}"; - }} - """ - ctx.info("Creating repository directory structure ...") - create_repo_path = _create_top_level_repo_path( - ctx, - repo_path, - salt_version, - distro, - distro_version=distro_version, - distro_arch=distro_arch, - nightly_build_from=nightly_build_from, - ) - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - - create_repo_path = _create_repo_path( - ctx, - repo_path, - salt_version, - distro, - distro_version=distro_version, - distro_arch=distro_arch, - nightly_build_from=nightly_build_from, - ) - ftp_archive_config_file = create_repo_path / "apt-ftparchive.conf" - ctx.info(f"Writing {ftp_archive_config_file} ...") - ftp_archive_config_file.write_text(textwrap.dedent(ftp_archive_config)) - - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - - pool_path = create_repo_path / "pool" - pool_path.mkdir(exist_ok=True) - for fpath in incoming.iterdir(): - dpath = pool_path / fpath.name - ctx.info(f"Copying {fpath} to {dpath} ...") - shutil.copyfile(fpath, dpath) - if fpath.suffix == ".dsc": - ctx.info(f"Running 'debsign' on {dpath} ...") - ctx.run("debsign", "--re-sign", "-k", key_id, str(dpath), interactive=True) - - dists_path = create_repo_path / "dists" - symlink_parent_path = dists_path / codename / "main" - symlink_paths = ( - symlink_parent_path / "by-hash" / "SHA256", - symlink_parent_path / "source" / "by-hash" / "SHA256", - symlink_parent_path / f"binary-{distro_arch}" / "by-hash" / "SHA256", - ) - - for path in symlink_paths: - path.mkdir(exist_ok=True, parents=True) - - cmdline = ["apt-ftparchive", "generate", "apt-ftparchive.conf"] - ctx.info(f"Running '{' '.join(cmdline)}' ...") - ctx.run(*cmdline, cwd=create_repo_path) - - ctx.info("Creating by-hash symlinks ...") - for path in symlink_paths: - for fpath in path.parent.parent.iterdir(): - if not fpath.is_file(): - continue - sha256sum = ctx.run("sha256sum", str(fpath), capture=True) - link = path / sha256sum.stdout.decode().split()[0] - link.symlink_to(f"../../{fpath.name}") - - cmdline = [ - "apt-ftparchive", - "--no-md5", - "--no-sha1", - "--no-sha512", - "release", - "-c", - "apt-ftparchive.conf", - f"dists/{codename}/", - ] - ctx.info(f"Running '{' '.join(cmdline)}' ...") - ret = ctx.run(*cmdline, capture=True, cwd=create_repo_path) - release_file = dists_path / codename / "Release" - ctx.info(f"Writing {release_file} with the output of the previous command...") - release_file.write_bytes(ret.stdout) - - cmdline = [ - "gpg", - "-u", - key_id, - "-o", - f"dists/{codename}/InRelease", - "-a", - "-s", - "--clearsign", - f"dists/{codename}/Release", - ] - ctx.info(f"Running '{' '.join(cmdline)}' ...") - ctx.run(*cmdline, cwd=create_repo_path) - - cmdline = [ - "gpg", - "-u", - key_id, - "-o", - f"dists/{codename}/Release.gpg", - "-a", - "-b", - "-s", - f"dists/{codename}/Release", - ] - - ctx.info(f"Running '{' '.join(cmdline)}' ...") - ctx.run(*cmdline, cwd=create_repo_path) - if not nightly_build_from: - remote_versions = _get_remote_versions( - tools.utils.STAGING_BUCKET_NAME, - create_repo_path.parent.relative_to(repo_path), - ) - major_version = Version(salt_version).major - matching_major = None - for version in remote_versions: - if version.major == major_version: - matching_major = version - break - if not matching_major or matching_major <= salt_version: - major_link = create_repo_path.parent.parent / str(major_version) - ctx.info(f"Creating '{major_link.relative_to(repo_path)}' symlink ...") - major_link.symlink_to(f"minor/{salt_version}") - if not remote_versions or remote_versions[0] <= salt_version: - latest_link = create_repo_path.parent.parent / "latest" - ctx.info(f"Creating '{latest_link.relative_to(repo_path)}' symlink ...") - latest_link.symlink_to(f"minor/{salt_version}") - - ctx.info("Done") - - -_rpm_distro_info = { - "amazon": ["2"], - "redhat": ["7", "8", "9"], - "fedora": ["36", "37", "38"], - "photon": ["3", "4"], -} - - -@create.command( - name="rpm", - arguments={ - "salt_version": { - "help": ( - "The salt version for which to build the repository configuration files. " - "If not passed, it will be discovered by running 'python3 salt/version.py'." - ), - "required": True, - }, - "distro": { - "help": "The debian based distribution to build the repository for", - "choices": list(_rpm_distro_info), - "required": True, - }, - "distro_version": { - "help": "The distro version.", - "required": True, - }, - "distro_arch": { - "help": "The distribution architecture", - "choices": ("x86_64", "aarch64", "arm64"), - }, - "repo_path": { - "help": "Path where the repository shall be created.", - "required": True, - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "incoming": { - "help": ( - "The path to the directory containing the files that should added to " - "the repository." - ), - "required": True, - }, - "nightly_build_from": { - "help": "Developement repository target", - }, - }, -) -def rpm( - ctx: Context, - salt_version: str = None, - distro: str = None, - distro_version: str = None, - incoming: pathlib.Path = None, - repo_path: pathlib.Path = None, - key_id: str = None, - distro_arch: str = "amd64", - nightly_build_from: str = None, -): - """ - Create the redhat repository. - """ - if TYPE_CHECKING: - assert salt_version is not None - assert distro is not None - assert distro_version is not None - assert incoming is not None - assert repo_path is not None - assert key_id is not None - display_name = f"{distro.capitalize()} {distro_version}" - if distro_version not in _rpm_distro_info[distro]: - ctx.error(f"Support for {display_name} is missing.") - ctx.exit(1) - - if distro_arch == "aarch64": - ctx.info(f"The {distro_arch} arch is an alias for 'arm64'. Adjusting.") - distro_arch = "arm64" - - ctx.info("Creating repository directory structure ...") - create_repo_path = _create_top_level_repo_path( - ctx, - repo_path, - salt_version, - distro, - distro_version=distro_version, - distro_arch=distro_arch, - nightly_build_from=nightly_build_from, - ) - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - - create_repo_path = _create_repo_path( - ctx, - repo_path, - salt_version, - distro, - distro_version=distro_version, - distro_arch=distro_arch, - nightly_build_from=nightly_build_from, - ) - - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - - for fpath in incoming.iterdir(): - if ".src" in fpath.suffixes: - dpath = create_repo_path / "SRPMS" / fpath.name - else: - dpath = create_repo_path / fpath.name - ctx.info(f"Copying {fpath} to {dpath} ...") - shutil.copyfile(fpath, dpath) - if fpath.suffix == ".rpm": - ctx.info(f"Running 'rpmsign' on {dpath} ...") - ctx.run( - "rpmsign", - "--key-id", - key_id, - "--addsign", - "--digest-algo=sha256", - str(dpath), - ) - - createrepo = shutil.which("createrepo") - if createrepo is None: - container = "ghcr.io/saltstack/salt-ci-containers/packaging:centosstream-9" - ctx.info(f"Using docker container '{container}' to call 'createrepo'...") - uid = ctx.run("id", "-u", capture=True).stdout.strip().decode() - gid = ctx.run("id", "-g", capture=True).stdout.strip().decode() - ctx.run( - "docker", - "run", - "--rm", - "-v", - f"{create_repo_path.resolve()}:/code", - "-u", - f"{uid}:{gid}", - "-w", - "/code", - container, - "createrepo", - ".", - ) - else: - ctx.run("createrepo", ".", cwd=create_repo_path) - - if nightly_build_from: - repo_domain = os.environ.get("SALT_REPO_DOMAIN_RELEASE", "repo.saltproject.io") - else: - repo_domain = os.environ.get( - "SALT_REPO_DOMAIN_STAGING", "staging.repo.saltproject.io" - ) - - salt_repo_user = os.environ.get("SALT_REPO_USER") - if salt_repo_user: - log.info( - "SALT_REPO_USER: %s", - salt_repo_user[0] + "*" * (len(salt_repo_user) - 2) + salt_repo_user[-1], - ) - salt_repo_pass = os.environ.get("SALT_REPO_PASS") - if salt_repo_pass: - log.info( - "SALT_REPO_PASS: %s", - salt_repo_pass[0] + "*" * (len(salt_repo_pass) - 2) + salt_repo_pass[-1], - ) - if salt_repo_user and salt_repo_pass: - repo_domain = f"{salt_repo_user}:{salt_repo_pass}@{repo_domain}" - - def _create_repo_file(create_repo_path, url_suffix): - ctx.info(f"Creating '{repo_file_path.relative_to(repo_path)}' file ...") - if nightly_build_from: - base_url = f"salt-dev/{nightly_build_from}/" - repo_file_contents = "[salt-nightly-repo]" - elif "rc" in salt_version: - base_url = "salt_rc/" - repo_file_contents = "[salt-rc-repo]" - else: - base_url = "" - repo_file_contents = "[salt-repo]" - base_url += f"salt/py3/{distro}/{distro_version}/{distro_arch}/{url_suffix}" - if distro == "amazon": - distro_name = "Amazon Linux" - elif distro == "redhat": - distro_name = "RHEL/CentOS" - else: - distro_name = distro.capitalize() - - if distro != "photon" and int(distro_version) < 8: - failovermethod = "\n failovermethod=priority" - else: - failovermethod = "" - - repo_file_contents += textwrap.dedent( - f""" - name=Salt repo for {distro_name} {distro_version} PY3 - baseurl=https://{repo_domain}/{base_url} - skip_if_unavailable=True{failovermethod} - priority=10 - enabled=1 - enabled_metadata=1 - gpgcheck=1 - gpgkey=https://{repo_domain}/{base_url}/{tools.utils.GPG_KEY_FILENAME}.pub - """ - ) - create_repo_path.write_text(repo_file_contents) - - if nightly_build_from: - repo_file_path = create_repo_path.parent / "nightly.repo" - else: - repo_file_path = create_repo_path.parent / f"{create_repo_path.name}.repo" - - _create_repo_file(repo_file_path, f"minor/{salt_version}") - - if not nightly_build_from: - remote_versions = _get_remote_versions( - tools.utils.STAGING_BUCKET_NAME, - create_repo_path.parent.relative_to(repo_path), - ) - major_version = Version(salt_version).major - matching_major = None - for version in remote_versions: - if version.major == major_version: - matching_major = version - break - if not matching_major or matching_major <= salt_version: - major_link = create_repo_path.parent.parent / str(major_version) - ctx.info(f"Creating '{major_link.relative_to(repo_path)}' symlink ...") - major_link.symlink_to(f"minor/{salt_version}") - repo_file_path = create_repo_path.parent.parent / f"{major_version}.repo" - _create_repo_file(repo_file_path, str(major_version)) - if not remote_versions or remote_versions[0] <= salt_version: - latest_link = create_repo_path.parent.parent / "latest" - ctx.info(f"Creating '{latest_link.relative_to(repo_path)}' symlink ...") - latest_link.symlink_to(f"minor/{salt_version}") - repo_file_path = create_repo_path.parent.parent / "latest.repo" - _create_repo_file(repo_file_path, "latest") - - ctx.info("Done") - - -@create.command( - name="windows", - arguments={ - "salt_version": { - "help": "The salt version for which to build the repository", - "required": True, - }, - "repo_path": { - "help": "Path where the repository shall be created.", - "required": True, - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "incoming": { - "help": ( - "The path to the directory containing the files that should added to " - "the repository." - ), - "required": True, - }, - "nightly_build_from": { - "help": "Developement repository target", - }, - }, -) -def windows( - ctx: Context, - salt_version: str = None, - incoming: pathlib.Path = None, - repo_path: pathlib.Path = None, - key_id: str = None, - nightly_build_from: str = None, -): - """ - Create the windows repository. - """ - if TYPE_CHECKING: - assert salt_version is not None - assert incoming is not None - assert repo_path is not None - assert key_id is not None - _create_onedir_based_repo( - ctx, - salt_version=salt_version, - nightly_build_from=nightly_build_from, - repo_path=repo_path, - incoming=incoming, - key_id=key_id, - distro="windows", - pkg_suffixes=(".msi", ".exe"), - ) - ctx.info("Done") - - -@create.command( - name="macos", - arguments={ - "salt_version": { - "help": "The salt version for which to build the repository", - "required": True, - }, - "repo_path": { - "help": "Path where the repository shall be created.", - "required": True, - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "incoming": { - "help": ( - "The path to the directory containing the files that should added to " - "the repository." - ), - "required": True, - }, - "nightly_build_from": { - "help": "Developement repository target", - }, - }, -) -def macos( - ctx: Context, - salt_version: str = None, - incoming: pathlib.Path = None, - repo_path: pathlib.Path = None, - key_id: str = None, - nightly_build_from: str = None, -): - """ - Create the windows repository. - """ - if TYPE_CHECKING: - assert salt_version is not None - assert incoming is not None - assert repo_path is not None - assert key_id is not None - _create_onedir_based_repo( - ctx, - salt_version=salt_version, - nightly_build_from=nightly_build_from, - repo_path=repo_path, - incoming=incoming, - key_id=key_id, - distro="macos", - pkg_suffixes=(".pkg",), - ) - ctx.info("Done") - - -@create.command( - name="onedir", - arguments={ - "salt_version": { - "help": "The salt version for which to build the repository", - "required": True, - }, - "repo_path": { - "help": "Path where the repository shall be created.", - "required": True, - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "incoming": { - "help": ( - "The path to the directory containing the files that should added to " - "the repository." - ), - "required": True, - }, - "nightly_build_from": { - "help": "Developement repository target", - }, - }, -) -def onedir( - ctx: Context, - salt_version: str = None, - incoming: pathlib.Path = None, - repo_path: pathlib.Path = None, - key_id: str = None, - nightly_build_from: str = None, -): - """ - Create the onedir repository. - """ - if TYPE_CHECKING: - assert salt_version is not None - assert incoming is not None - assert repo_path is not None - assert key_id is not None - _create_onedir_based_repo( - ctx, - salt_version=salt_version, - nightly_build_from=nightly_build_from, - repo_path=repo_path, - incoming=incoming, - key_id=key_id, - distro="onedir", - pkg_suffixes=(".xz", ".zip"), - ) - ctx.info("Done") - - -@create.command( - name="src", - arguments={ - "salt_version": { - "help": "The salt version for which to build the repository", - "required": True, - }, - "repo_path": { - "help": "Path where the repository shall be created.", - "required": True, - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "incoming": { - "help": ( - "The path to the directory containing the files that should added to " - "the repository." - ), - "required": True, - }, - "nightly_build_from": { - "help": "Developement repository target", - }, - }, -) -def src( - ctx: Context, - salt_version: str = None, - incoming: pathlib.Path = None, - repo_path: pathlib.Path = None, - key_id: str = None, - nightly_build_from: str = None, -): - """ - Create the onedir repository. - """ - if TYPE_CHECKING: - assert salt_version is not None - assert incoming is not None - assert repo_path is not None - assert key_id is not None - - ctx.info("Creating repository directory structure ...") - create_repo_path = _create_top_level_repo_path( - ctx, - repo_path, - salt_version, - distro="src", - nightly_build_from=nightly_build_from, - ) - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - create_repo_path = create_repo_path / salt_version - create_repo_path.mkdir(exist_ok=True, parents=True) - hashes_base_path = create_repo_path / f"salt-{salt_version}" - for fpath in incoming.iterdir(): - if fpath.suffix not in (".gz",): - continue - ctx.info(f"* Processing {fpath} ...") - dpath = create_repo_path / fpath.name - ctx.info(f"Copying {fpath} to {dpath} ...") - shutil.copyfile(fpath, dpath) - for hash_name in ("blake2b", "sha512", "sha3_512"): - ctx.info(f" * Calculating {hash_name} ...") - hexdigest = _get_file_checksum(fpath, hash_name) - with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh: - wfh.write(f"{hexdigest} {dpath.name}\n") - with open(f"{dpath}.{hash_name}", "a+") as wfh: - wfh.write(f"{hexdigest} {dpath.name}\n") - - for fpath in create_repo_path.iterdir(): - if fpath.suffix in (".pub", ".gpg"): - continue - tools.utils.gpg_sign(ctx, key_id, fpath) - - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - ctx.info("Done") - - -@publish.command( - arguments={ - "repo_path": { - "help": "Local path for the repository that shall be published.", - }, - "salt_version": { - "help": "The salt version for which to build the repository", - "required": True, - }, - } -) -def nightly(ctx: Context, repo_path: pathlib.Path, salt_version: str = None): - """ - Publish to the nightly bucket. - """ - if TYPE_CHECKING: - assert salt_version is not None - _publish_repo( - ctx, repo_path=repo_path, nightly_build=True, salt_version=salt_version - ) - - -@publish.command( - arguments={ - "repo_path": { - "help": "Local path for the repository that shall be published.", - }, - "salt_version": { - "help": "The salt version for which to build the repository", - "required": True, - }, - } -) -def staging(ctx: Context, repo_path: pathlib.Path, salt_version: str = None): - """ - Publish to the staging bucket. - """ - if TYPE_CHECKING: - assert salt_version is not None - _publish_repo(ctx, repo_path=repo_path, stage=True, salt_version=salt_version) - - -@repo.command(name="backup-previous-releases") -def backup_previous_releases(ctx: Context): - """ - Backup release bucket. - """ - _rclone(ctx, tools.utils.RELEASE_BUCKET_NAME, tools.utils.BACKUP_BUCKET_NAME) - ctx.info("Done") - - -@repo.command(name="restore-previous-releases") -def restore_previous_releases(ctx: Context): - """ - Restore release bucket from backup. - """ - _rclone(ctx, tools.utils.BACKUP_BUCKET_NAME, tools.utils.RELEASE_BUCKET_NAME) - github_output = os.environ.get("GITHUB_OUTPUT") - if github_output is not None: - with open(github_output, "a", encoding="utf-8") as wfh: - wfh.write(f"backup-complete=true\n") - ctx.info("Done") - - -def _rclone(ctx: Context, src: str, dst: str): - rclone = shutil.which("rclone") - if not rclone: - ctx.error("Could not find the rclone binary") - ctx.exit(1) - - if TYPE_CHECKING: - assert rclone - - env = os.environ.copy() - env["RCLONE_CONFIG_S3_TYPE"] = "s3" - cmdline: list[str] = [ - rclone, - "sync", - "--auto-confirm", - "--human-readable", - "--checksum", - "--color=always", - "--metadata", - "--s3-env-auth", - "--s3-location-constraint=us-west-2", - "--s3-provider=AWS", - "--s3-region=us-west-2", - "--stats-file-name-length=0", - "--stats-one-line", - "--stats=5s", - "--transfers=50", - "--fast-list", - "--verbose", - ] - if src == tools.utils.RELEASE_BUCKET_NAME: - cmdline.append("--s3-storage-class=INTELLIGENT_TIERING") - cmdline.extend([f"s3://{src}", f"s3://{dst}"]) - ctx.info(f"Running: {' '.join(cmdline)}") - ret = ctx.run(*cmdline, env=env, check=False) - if ret.returncode: - ctx.error(f"Failed to sync from s3://{src} to s3://{dst}") - ctx.exit(1) - - -@publish.command( - arguments={ - "salt_version": { - "help": "The salt version to release.", - }, - } -) -def release(ctx: Context, salt_version: str): - """ - Publish to the release bucket. - """ - if "rc" in salt_version: - bucket_folder = "salt_rc/salt/py3" - else: - bucket_folder = "salt/py3" - - files_to_copy: list[str] - directories_to_delete: list[str] = [] - - ctx.info("Grabbing remote file listing of files to copy...") - s3 = boto3.client("s3") - repo_release_files_path = pathlib.Path( - f"release-artifacts/{salt_version}/.release-files.json" - ) - repo_release_symlinks_path = pathlib.Path( - f"release-artifacts/{salt_version}/.release-symlinks.json" - ) - with tempfile.TemporaryDirectory(prefix=f"{salt_version}_release_") as tsd: - local_release_files_path = pathlib.Path(tsd) / repo_release_files_path.name - try: - bucket_name = tools.utils.STAGING_BUCKET_NAME - with local_release_files_path.open("wb") as wfh: - ctx.info( - f"Downloading {repo_release_files_path} from bucket {bucket_name} ..." - ) - s3.download_fileobj( - Bucket=bucket_name, - Key=str(repo_release_files_path), - Fileobj=wfh, - ) - files_to_copy = json.loads(local_release_files_path.read_text()) - except ClientError as exc: - if "Error" not in exc.response: - log.exception(f"Error downloading {repo_release_files_path}: {exc}") - ctx.exit(1) - if exc.response["Error"]["Code"] == "404": - ctx.error(f"Could not find {repo_release_files_path} in bucket.") - ctx.exit(1) - if exc.response["Error"]["Code"] == "400": - ctx.error( - f"Could not download {repo_release_files_path} from bucket: {exc}" - ) - ctx.exit(1) - log.exception(f"Error downloading {repo_release_files_path}: {exc}") - ctx.exit(1) - local_release_symlinks_path = ( - pathlib.Path(tsd) / repo_release_symlinks_path.name - ) - try: - with local_release_symlinks_path.open("wb") as wfh: - ctx.info( - f"Downloading {repo_release_symlinks_path} from bucket {bucket_name} ..." - ) - s3.download_fileobj( - Bucket=bucket_name, - Key=str(repo_release_symlinks_path), - Fileobj=wfh, - ) - directories_to_delete = json.loads(local_release_symlinks_path.read_text()) - except ClientError as exc: - if "Error" not in exc.response: - log.exception(f"Error downloading {repo_release_symlinks_path}: {exc}") - ctx.exit(1) - if exc.response["Error"]["Code"] == "404": - ctx.error(f"Could not find {repo_release_symlinks_path} in bucket.") - ctx.exit(1) - if exc.response["Error"]["Code"] == "400": - ctx.error( - f"Could not download {repo_release_symlinks_path} from bucket: {exc}" - ) - ctx.exit(1) - log.exception(f"Error downloading {repo_release_symlinks_path}: {exc}") - ctx.exit(1) - - if directories_to_delete: - with tools.utils.create_progress_bar() as progress: - task = progress.add_task( - "Deleting directories to override.", - total=len(directories_to_delete), - ) - for directory in directories_to_delete: - try: - objects_to_delete: list[dict[str, str]] = [] - for path in _get_repo_file_list( - bucket_name=tools.utils.RELEASE_BUCKET_NAME, - bucket_folder=bucket_folder, - glob_match=f"{directory}/**", - ): - objects_to_delete.append({"Key": path}) - if objects_to_delete: - s3.delete_objects( - Bucket=tools.utils.RELEASE_BUCKET_NAME, - Delete={"Objects": objects_to_delete}, - ) - except ClientError: - log.exception("Failed to delete remote files") - finally: - progress.update(task, advance=1) - - already_copied_files: list[str] = [] - s3 = boto3.client("s3") - dot_repo_files = [] - with tools.utils.create_progress_bar() as progress: - task = progress.add_task( - "Copying files between buckets", total=len(files_to_copy) - ) - for fpath in files_to_copy: - if fpath in already_copied_files: - continue - if fpath.endswith(".repo"): - dot_repo_files.append(fpath) - ctx.info(f" * Copying {fpath}") - try: - s3.copy_object( - Bucket=tools.utils.RELEASE_BUCKET_NAME, - Key=fpath, - CopySource={ - "Bucket": tools.utils.STAGING_BUCKET_NAME, - "Key": fpath, - }, - MetadataDirective="COPY", - TaggingDirective="COPY", - ServerSideEncryption="AES256", - ) - already_copied_files.append(fpath) - except ClientError: - log.exception(f"Failed to copy {fpath}") - finally: - progress.update(task, advance=1) - - # Now let's get the onedir based repositories where we need to update several repo.json - major_version = packaging.version.parse(salt_version).major - with tempfile.TemporaryDirectory(prefix=f"{salt_version}_release_") as tsd: - repo_path = pathlib.Path(tsd) - for distro in ("windows", "macos", "onedir"): - - create_repo_path = _create_repo_path( - ctx, - repo_path, - salt_version, - distro=distro, - ) - repo_json_path = create_repo_path.parent.parent / "repo.json" - - release_repo_json = _get_repo_json_file_contents( - ctx, - bucket_name=tools.utils.RELEASE_BUCKET_NAME, - repo_path=repo_path, - repo_json_path=repo_json_path, - ) - minor_repo_json_path = create_repo_path.parent / "repo.json" - - staging_minor_repo_json = _get_repo_json_file_contents( - ctx, - bucket_name=tools.utils.STAGING_BUCKET_NAME, - repo_path=repo_path, - repo_json_path=minor_repo_json_path, - ) - release_minor_repo_json = _get_repo_json_file_contents( - ctx, - bucket_name=tools.utils.RELEASE_BUCKET_NAME, - repo_path=repo_path, - repo_json_path=minor_repo_json_path, - ) - - release_json = staging_minor_repo_json[salt_version] - - major_version = Version(salt_version).major - versions = _parse_versions(*list(release_minor_repo_json)) - ctx.info( - f"Collected versions from {minor_repo_json_path.relative_to(repo_path)}: " - f"{', '.join(str(vs) for vs in versions)}" - ) - minor_versions = [v for v in versions if v.major == major_version] - ctx.info( - f"Collected versions(Matching major: {major_version}) from " - f"{minor_repo_json_path.relative_to(repo_path)}: " - f"{', '.join(str(vs) for vs in minor_versions)}" - ) - if not versions: - latest_version = Version(salt_version) - else: - latest_version = versions[0] - if not minor_versions: - latest_minor_version = Version(salt_version) - else: - latest_minor_version = minor_versions[0] - - ctx.info(f"Release Version: {salt_version}") - ctx.info(f"Latest Repo Version: {latest_version}") - ctx.info(f"Latest Release Minor Version: {latest_minor_version}") - - # Add the minor version - release_minor_repo_json[salt_version] = release_json - - if latest_version <= salt_version: - release_repo_json["latest"] = release_json - - if latest_minor_version <= salt_version: - release_minor_repo_json["latest"] = release_json - - ctx.info(f"Writing {minor_repo_json_path} ...") - minor_repo_json_path.write_text( - json.dumps(release_minor_repo_json, sort_keys=True) - ) - ctx.info(f"Writing {repo_json_path} ...") - repo_json_path.write_text(json.dumps(release_repo_json, sort_keys=True)) - - # And now, let's get the several rpm "*.repo" files to update the base - # domain from staging to release - release_domain = os.environ.get( - "SALT_REPO_DOMAIN_RELEASE", "repo.saltproject.io" - ) - for path in dot_repo_files: - repo_file_path = repo_path.joinpath(path) - repo_file_path.parent.mkdir(exist_ok=True, parents=True) - bucket_name = tools.utils.STAGING_BUCKET_NAME - try: - ret = s3.head_object(Bucket=bucket_name, Key=path) - ctx.info( - f"Downloading existing '{repo_file_path.relative_to(repo_path)}' " - f"file from bucket {bucket_name}" - ) - size = ret["ContentLength"] - with repo_file_path.open("wb") as wfh: - with tools.utils.create_progress_bar( - file_progress=True - ) as progress: - task = progress.add_task( - description="Downloading...", total=size - ) - s3.download_fileobj( - Bucket=bucket_name, - Key=path, - Fileobj=wfh, - Callback=tools.utils.UpdateProgress(progress, task), - ) - updated_contents = re.sub( - r"^(baseurl|gpgkey)=https://([^/]+)/(.*)$", - rf"\1=https://{release_domain}/\3", - repo_file_path.read_text(), - flags=re.MULTILINE, - ) - ctx.info(f"Updated '{repo_file_path.relative_to(repo_path)}:") - ctx.print(updated_contents) - repo_file_path.write_text(updated_contents) - except ClientError as exc: - if "Error" not in exc.response: - raise - if exc.response["Error"]["Code"] != "404": - raise - ctx.info(f"Could not find {repo_file_path} in bucket {bucket_name}") - - for dirpath, dirnames, filenames in os.walk(repo_path, followlinks=True): - for path in filenames: - upload_path = pathlib.Path(dirpath, path) - relpath = upload_path.relative_to(repo_path) - size = upload_path.stat().st_size - ctx.info(f" {relpath}") - with tools.utils.create_progress_bar(file_progress=True) as progress: - task = progress.add_task(description="Uploading...", total=size) - s3.upload_file( - str(upload_path), - tools.utils.RELEASE_BUCKET_NAME, - str(relpath), - Callback=tools.utils.UpdateProgress(progress, task), - ) - - -@publish.command( - arguments={ - "salt_version": { - "help": "The salt version to release.", - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "repository": { - "help": ( - "The full repository name, ie, 'saltstack/salt' on GitHub " - "to run the checks against." - ) - }, - } -) -def github( - ctx: Context, - salt_version: str, - key_id: str = None, - repository: str = "saltstack/salt", -): - """ - Publish the release on GitHub releases. - """ - if TYPE_CHECKING: - assert key_id is not None - - s3 = boto3.client("s3") - - # Let's download the release artifacts stored in staging - artifacts_path = pathlib.Path.cwd() / "release-artifacts" - artifacts_path.mkdir(exist_ok=True) - release_artifacts_listing: dict[pathlib.Path, int] = {} - continuation_token = None - while True: - kwargs: dict[str, str] = {} - if continuation_token: - kwargs["ContinuationToken"] = continuation_token - ret = s3.list_objects_v2( - Bucket=tools.utils.STAGING_BUCKET_NAME, - Prefix=f"release-artifacts/{salt_version}", - FetchOwner=False, - **kwargs, - ) - contents = ret.pop("Contents", None) - if contents is None: - break - for entry in contents: - entry_path = pathlib.Path(entry["Key"]) - if entry_path.name.startswith("."): - continue - release_artifacts_listing[entry_path] = entry["Size"] - if not ret["IsTruncated"]: - break - continuation_token = ret["NextContinuationToken"] - - for entry_path, size in release_artifacts_listing.items(): - ctx.info(f" * {entry_path.name}") - local_path = artifacts_path / entry_path.name - with local_path.open("wb") as wfh: - with tools.utils.create_progress_bar(file_progress=True) as progress: - task = progress.add_task(description="Downloading...", total=size) - s3.download_fileobj( - Bucket=tools.utils.STAGING_BUCKET_NAME, - Key=str(entry_path), - Fileobj=wfh, - Callback=tools.utils.UpdateProgress(progress, task), - ) - - for artifact in artifacts_path.iterdir(): - if artifact.suffix in (".patch", ".asc", ".gpg", ".pub"): - continue - tools.utils.gpg_sign(ctx, key_id, artifact) - - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, artifacts_path) - - release_message = f"""\ - # Welcome to Salt v{salt_version} - - | :exclamation: ATTENTION | - |:-------------------------------------------------------------------------------------------------------------------------| - | The archives generated by GitHub(`Source code(zip)`, `Source code(tar.gz)`) will not report Salt's version properly. | - | Please use the tarball generated by The Salt Project Team(`salt-{salt_version}.tar.gz`). - """ - release_message_path = artifacts_path / "gh-release-body.md" - release_message_path.write_text(textwrap.dedent(release_message).strip()) - - github_output = os.environ.get("GITHUB_OUTPUT") - if github_output is None: - ctx.warn("The 'GITHUB_OUTPUT' variable is not set. Stop processing.") - ctx.exit(0) - - if TYPE_CHECKING: - assert github_output is not None - - with open(github_output, "a", encoding="utf-8") as wfh: - wfh.write(f"release-messsage-file={release_message_path.resolve()}\n") - - releases = get_salt_releases(ctx, repository) - if Version(salt_version) >= releases[-1]: - make_latest = True - else: - make_latest = False - with open(github_output, "a", encoding="utf-8") as wfh: - wfh.write(f"make-latest={json.dumps(make_latest)}\n") - - artifacts_to_upload = [] - for artifact in artifacts_path.iterdir(): - if artifact.suffix == ".patch": - continue - if artifact.name == release_message_path.name: - continue - artifacts_to_upload.append(str(artifact.resolve())) - - with open(github_output, "a", encoding="utf-8") as wfh: - wfh.write(f"release-artifacts={','.join(artifacts_to_upload)}\n") - ctx.exit(0) - - -@repo.command( - name="confirm-unreleased", - arguments={ - "salt_version": { - "help": "The salt version to check", - }, - "repository": { - "help": ( - "The full repository name, ie, 'saltstack/salt' on GitHub " - "to run the checks against." - ) - }, - }, -) -def confirm_unreleased( - ctx: Context, salt_version: str, repository: str = "saltstack/salt" -): - """ - Confirm that the passed version is not yet tagged and/or released. - """ - releases = get_salt_releases(ctx, repository) - if Version(salt_version) in releases: - ctx.error(f"There's already a '{salt_version}' tag or github release.") - ctx.exit(1) - ctx.info(f"Could not find a release for Salt Version '{salt_version}'") - ctx.exit(0) - - -@repo.command( - name="confirm-staged", - arguments={ - "salt_version": { - "help": "The salt version to check", - }, - "repository": { - "help": ( - "The full repository name, ie, 'saltstack/salt' on GitHub " - "to run the checks against." - ) - }, - }, -) -def confirm_staged(ctx: Context, salt_version: str, repository: str = "saltstack/salt"): - """ - Confirm that the passed version has been staged for release. - """ - s3 = boto3.client("s3") - repo_release_files_path = pathlib.Path( - f"release-artifacts/{salt_version}/.release-files.json" - ) - repo_release_symlinks_path = pathlib.Path( - f"release-artifacts/{salt_version}/.release-symlinks.json" - ) - for remote_path in (repo_release_files_path, repo_release_symlinks_path): - try: - bucket_name = tools.utils.STAGING_BUCKET_NAME - ctx.info( - f"Checking for the presence of {remote_path} on bucket {bucket_name} ..." - ) - s3.head_object( - Bucket=bucket_name, - Key=str(remote_path), - ) - except ClientError as exc: - if "Error" not in exc.response: - log.exception(f"Could not get information about {remote_path}: {exc}") - ctx.exit(1) - if exc.response["Error"]["Code"] == "404": - ctx.error(f"Could not find {remote_path} in bucket.") - ctx.exit(1) - if exc.response["Error"]["Code"] == "400": - ctx.error(f"Could get information about {remote_path}: {exc}") - ctx.exit(1) - log.exception(f"Error getting information about {remote_path}: {exc}") - ctx.exit(1) - ctx.info(f"Version {salt_version} has been staged for release") - ctx.exit(0) - - -def _get_repo_detailed_file_list( - bucket_name: str, - bucket_folder: str = "", - glob_match: str = "**", -) -> list[dict[str, Any]]: - s3 = boto3.client("s3") - listing: list[dict[str, Any]] = [] - continuation_token = None - while True: - kwargs: dict[str, str] = {} - if continuation_token: - kwargs["ContinuationToken"] = continuation_token - ret = s3.list_objects_v2( - Bucket=bucket_name, - Prefix=bucket_folder, - FetchOwner=False, - **kwargs, - ) - contents = ret.pop("Contents", None) - if contents is None: - break - for entry in contents: - if fnmatch.fnmatch(entry["Key"], glob_match): - listing.append(entry) - if not ret["IsTruncated"]: - break - continuation_token = ret["NextContinuationToken"] - return listing - - -def _get_repo_file_list( - bucket_name: str, bucket_folder: str, glob_match: str -) -> list[str]: - return [ - entry["Key"] - for entry in _get_repo_detailed_file_list( - bucket_name, bucket_folder, glob_match=glob_match - ) - ] - - -def _get_remote_versions(bucket_name: str, remote_path: str): - log.info( - "Getting remote versions from bucket %r under path: %s", - bucket_name, - remote_path, - ) - remote_path = str(remote_path) - if not remote_path.endswith("/"): - remote_path += "/" - - s3 = boto3.client("s3") - ret = s3.list_objects( - Bucket=bucket_name, - Delimiter="/", - Prefix=remote_path, - ) - if "CommonPrefixes" not in ret: - return [] - versions = [] - for entry in ret["CommonPrefixes"]: - _, version = entry["Prefix"].rstrip("/").rsplit("/", 1) - if version == "latest": - continue - versions.append(Version(version)) - versions.sort(reverse=True) - log.info("Remote versions collected: %s", versions) - return versions - - -def _create_onedir_based_repo( - ctx: Context, - salt_version: str, - nightly_build_from: str | None, - repo_path: pathlib.Path, - incoming: pathlib.Path, - key_id: str, - distro: str, - pkg_suffixes: tuple[str, ...], -): - ctx.info("Creating repository directory structure ...") - create_repo_path = _create_top_level_repo_path( - ctx, - repo_path, - salt_version, - distro, - nightly_build_from=nightly_build_from, - ) - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - - create_repo_path = _create_repo_path( - ctx, - repo_path, - salt_version, - distro, - nightly_build_from=nightly_build_from, - ) - if not nightly_build_from: - repo_json_path = create_repo_path.parent.parent / "repo.json" - else: - repo_json_path = create_repo_path.parent / "repo.json" - - if nightly_build_from: - bucket_name = tools.utils.RELEASE_BUCKET_NAME - else: - bucket_name = tools.utils.STAGING_BUCKET_NAME - - release_json = {} - - copy_exclusions = ( - ".blake2b", - ".sha512", - ".sha3_512", - ".BLAKE2B", - ".SHA512", - ".SHA3_512", - ".json", - ) - hashes_base_path = create_repo_path / f"salt-{salt_version}" - for fpath in incoming.iterdir(): - if fpath.suffix in copy_exclusions: - continue - ctx.info(f"* Processing {fpath} ...") - dpath = create_repo_path / fpath.name - ctx.info(f"Copying {fpath} to {dpath} ...") - shutil.copyfile(fpath, dpath) - if "-amd64" in dpath.name.lower(): - arch = "amd64" - elif "-x86_64" in dpath.name.lower(): - arch = "x86_64" - elif "-x86" in dpath.name.lower(): - arch = "x86" - elif "-aarch64" in dpath.name.lower(): - arch = "aarch64" - else: - ctx.error( - f"Cannot pickup the right architecture from the filename '{dpath.name}'." - ) - ctx.exit(1) - if distro == "onedir": - if "-onedir-linux-" in dpath.name.lower(): - release_os = "linux" - elif "-onedir-darwin-" in dpath.name.lower(): - release_os = "macos" - elif "-onedir-windows-" in dpath.name.lower(): - release_os = "windows" - else: - ctx.error( - f"Cannot pickup the right OS from the filename '{dpath.name}'." - ) - ctx.exit(1) - else: - release_os = distro - release_json[dpath.name] = { - "name": dpath.name, - "version": salt_version, - "os": release_os, - "arch": arch, - } - for hash_name in ("blake2b", "sha512", "sha3_512"): - ctx.info(f" * Calculating {hash_name} ...") - hexdigest = _get_file_checksum(fpath, hash_name) - release_json[dpath.name][hash_name.upper()] = hexdigest - with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh: - wfh.write(f"{hexdigest} {dpath.name}\n") - with open(f"{dpath}.{hash_name}", "a+") as wfh: - wfh.write(f"{hexdigest} {dpath.name}\n") - - for fpath in create_repo_path.iterdir(): - if fpath.suffix in pkg_suffixes: - continue - tools.utils.gpg_sign(ctx, key_id, fpath) - - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - - repo_json = _get_repo_json_file_contents( - ctx, bucket_name=bucket_name, repo_path=repo_path, repo_json_path=repo_json_path - ) - if nightly_build_from: - ctx.info(f"Writing {repo_json_path} ...") - repo_json_path.write_text(json.dumps(repo_json, sort_keys=True)) - return - - major_version = Version(salt_version).major - minor_repo_json_path = create_repo_path.parent / "repo.json" - minor_repo_json = _get_repo_json_file_contents( - ctx, - bucket_name=bucket_name, - repo_path=repo_path, - repo_json_path=minor_repo_json_path, - ) - minor_repo_json[salt_version] = release_json - versions = _parse_versions(*list(minor_repo_json)) - ctx.info( - f"Collected versions from {minor_repo_json_path.relative_to(repo_path)}: " - f"{', '.join(str(vs) for vs in versions)}" - ) - minor_versions = [v for v in versions if v.major == major_version] - ctx.info( - f"Collected versions(Matching major: {major_version}) from " - f"{minor_repo_json_path.relative_to(repo_path)}: " - f"{', '.join(str(vs) for vs in minor_versions)}" - ) - if not versions: - latest_version = Version(salt_version) - else: - latest_version = versions[0] - if not minor_versions: - latest_minor_version = Version(salt_version) - else: - latest_minor_version = minor_versions[0] - - ctx.info(f"Release Version: {salt_version}") - ctx.info(f"Latest Repo Version: {latest_version}") - ctx.info(f"Latest Release Minor Version: {latest_minor_version}") - - latest_link = create_repo_path.parent.parent / "latest" - if latest_version <= salt_version: - repo_json["latest"] = release_json - ctx.info(f"Creating '{latest_link.relative_to(repo_path)}' symlink ...") - if latest_link.exists(): - latest_link.unlink() - latest_link.symlink_to(f"minor/{salt_version}") - else: - ctx.info( - f"Not creating the '{latest_link.relative_to(repo_path)}' symlink " - f"since {latest_version} > {salt_version}" - ) - - major_link = create_repo_path.parent.parent / str(major_version) - if latest_minor_version <= salt_version: - minor_repo_json["latest"] = release_json - # This is the latest minor, update the major in the top level repo.json - # to this version - repo_json[str(major_version)] = release_json - ctx.info(f"Creating '{major_link.relative_to(repo_path)}' symlink ...") - if major_link.exists(): - major_link.unlink() - major_link.symlink_to(f"minor/{salt_version}") - else: - ctx.info( - f"Not creating the '{major_link.relative_to(repo_path)}' symlink " - f"since {latest_minor_version} > {salt_version}" - ) - - ctx.info(f"Writing {minor_repo_json_path} ...") - minor_repo_json_path.write_text(json.dumps(minor_repo_json, sort_keys=True)) - - ctx.info(f"Writing {repo_json_path} ...") - repo_json_path.write_text(json.dumps(repo_json, sort_keys=True)) - - -def _get_repo_json_file_contents( - ctx: Context, - bucket_name: str, - repo_path: pathlib.Path, - repo_json_path: pathlib.Path, -) -> dict[str, Any]: - s3 = boto3.client("s3") - repo_json: dict[str, Any] = {} - try: - ret = s3.head_object( - Bucket=bucket_name, Key=str(repo_json_path.relative_to(repo_path)) - ) - ctx.info( - f"Downloading existing '{repo_json_path.relative_to(repo_path)}' file " - f"from bucket {bucket_name}" - ) - size = ret["ContentLength"] - with repo_json_path.open("wb") as wfh: - with tools.utils.create_progress_bar(file_progress=True) as progress: - task = progress.add_task(description="Downloading...", total=size) - s3.download_fileobj( - Bucket=bucket_name, - Key=str(repo_json_path.relative_to(repo_path)), - Fileobj=wfh, - Callback=tools.utils.UpdateProgress(progress, task), - ) - with repo_json_path.open() as rfh: - repo_json = json.load(rfh) - except ClientError as exc: - if "Error" not in exc.response: - raise - if exc.response["Error"]["Code"] != "404": - raise - ctx.info(f"Could not find {repo_json_path} in bucket {bucket_name}") - if repo_json: - ctx.print(repo_json, soft_wrap=True) - return repo_json - - -def _get_file_checksum(fpath: pathlib.Path, hash_name: str) -> str: - - with fpath.open("rb") as rfh: - try: - digest = hashlib.file_digest(rfh, hash_name) # type: ignore[attr-defined] - except AttributeError: - # Python < 3.11 - buf = bytearray(2**18) # Reusable buffer to reduce allocations. - view = memoryview(buf) - digest = getattr(hashlib, hash_name)() - while True: - size = rfh.readinto(buf) - if size == 0: - break # EOF - digest.update(view[:size]) - hexdigest: str = digest.hexdigest() - return hexdigest - - -def _publish_repo( - ctx: Context, - repo_path: pathlib.Path, - salt_version: str, - nightly_build: bool = False, - stage: bool = False, -): - """ - Publish packaging repositories. - """ - if nightly_build: - bucket_name = tools.utils.RELEASE_BUCKET_NAME - elif stage: - bucket_name = tools.utils.STAGING_BUCKET_NAME - else: - bucket_name = tools.utils.RELEASE_BUCKET_NAME - - ctx.info("Preparing upload ...") - s3 = boto3.client("s3") - to_delete_paths: dict[pathlib.Path, list[dict[str, str]]] = {} - to_upload_paths: list[pathlib.Path] = [] - symlink_paths: list[str] = [] - uploaded_files: list[str] = [] - for dirpath, dirnames, filenames in os.walk(repo_path, followlinks=True): - for dirname in dirnames: - path = pathlib.Path(dirpath, dirname) - if not path.is_symlink(): - continue - # This is a symlink, then we need to delete all files under - # that directory in S3 because S3 does not understand symlinks - # and we would end up adding files to that folder instead of - # replacing it. - try: - relpath = path.relative_to(repo_path) - ret = s3.list_objects( - Bucket=bucket_name, - Prefix=str(relpath), - ) - if "Contents" not in ret: - continue - objects = [] - for entry in ret["Contents"]: - objects.append({"Key": entry["Key"]}) - to_delete_paths[path] = objects - symlink_paths.append(str(relpath)) - except ClientError as exc: - if "Error" not in exc.response: - raise - if exc.response["Error"]["Code"] != "404": - raise - - for fpath in filenames: - path = pathlib.Path(dirpath, fpath) - to_upload_paths.append(path) - - with tools.utils.create_progress_bar() as progress: - task = progress.add_task( - "Deleting directories to override.", total=len(to_delete_paths) - ) - for base, objects in to_delete_paths.items(): - relpath = base.relative_to(repo_path) - bucket_uri = f"s3://{bucket_name}/{relpath}" - progress.update(task, description=f"Deleting {bucket_uri}") - try: - ret = s3.delete_objects( - Bucket=bucket_name, - Delete={"Objects": objects}, - ) - except ClientError: - log.exception(f"Failed to delete {bucket_uri}") - finally: - progress.update(task, advance=1) - - try: - ctx.info("Uploading repository ...") - for upload_path in to_upload_paths: - relpath = upload_path.relative_to(repo_path) - size = upload_path.stat().st_size - ctx.info(f" {relpath}") - with tools.utils.create_progress_bar(file_progress=True) as progress: - task = progress.add_task(description="Uploading...", total=size) - s3.upload_file( - str(upload_path), - bucket_name, - str(relpath), - Callback=tools.utils.UpdateProgress(progress, task), - ExtraArgs={ - "Metadata": { - "x-amz-meta-salt-release-version": salt_version, - } - }, - ) - uploaded_files.append(str(relpath)) - if stage is True: - repo_files_path = f"release-artifacts/{salt_version}/.release-files.json" - ctx.info(f"Uploading {repo_files_path} ...") - s3.put_object( - Key=repo_files_path, - Bucket=bucket_name, - Body=json.dumps(uploaded_files).encode(), - Metadata={ - "x-amz-meta-salt-release-version": salt_version, - }, - ) - repo_symlinks_path = ( - f"release-artifacts/{salt_version}/.release-symlinks.json" - ) - ctx.info(f"Uploading {repo_symlinks_path} ...") - s3.put_object( - Key=repo_symlinks_path, - Bucket=bucket_name, - Body=json.dumps(symlink_paths).encode(), - Metadata={ - "x-amz-meta-salt-release-version": salt_version, - }, - ) - except KeyboardInterrupt: - pass - - -def _create_top_level_repo_path( - ctx: Context, - repo_path: pathlib.Path, - salt_version: str, - distro: str, - distro_version: str | None = None, # pylint: disable=bad-whitespace - distro_arch: str | None = None, # pylint: disable=bad-whitespace - nightly_build_from: str | None = None, # pylint: disable=bad-whitespace -): - create_repo_path = repo_path - if nightly_build_from: - create_repo_path = ( - create_repo_path - / "salt-dev" - / nightly_build_from - / datetime.utcnow().strftime("%Y-%m-%d") - ) - create_repo_path.mkdir(exist_ok=True, parents=True) - with ctx.chdir(create_repo_path.parent): - latest_nightly_symlink = pathlib.Path("latest") - if not latest_nightly_symlink.exists(): - ctx.info( - f"Creating 'latest' symlink to '{create_repo_path.relative_to(repo_path)}' ..." - ) - latest_nightly_symlink.symlink_to( - create_repo_path.name, target_is_directory=True - ) - elif "rc" in salt_version: - create_repo_path = create_repo_path / "salt_rc" - create_repo_path = create_repo_path / "salt" / "py3" / distro - if distro_version: - create_repo_path = create_repo_path / distro_version - if distro_arch: - create_repo_path = create_repo_path / distro_arch - create_repo_path.mkdir(exist_ok=True, parents=True) - return create_repo_path - - -def _create_repo_path( - ctx: Context, - repo_path: pathlib.Path, - salt_version: str, - distro: str, - distro_version: str | None = None, # pylint: disable=bad-whitespace - distro_arch: str | None = None, # pylint: disable=bad-whitespace - nightly_build_from: str | None = None, # pylint: disable=bad-whitespace -): - create_repo_path = _create_top_level_repo_path( - ctx, - repo_path, - salt_version, - distro, - distro_version, - distro_arch, - nightly_build_from=nightly_build_from, - ) - create_repo_path = create_repo_path / "minor" / salt_version - create_repo_path.mkdir(exist_ok=True, parents=True) - return create_repo_path - - -def _parse_versions(*versions: str) -> list[Version]: - _versions = [] - for version in set(versions): - if version == "latest": - continue - _versions.append(Version(version)) - if _versions: - _versions.sort(reverse=True) - return _versions diff --git a/tools/pkg/repo/__init__.py b/tools/pkg/repo/__init__.py new file mode 100644 index 00000000000..8a3cbd9c81f --- /dev/null +++ b/tools/pkg/repo/__init__.py @@ -0,0 +1,181 @@ +""" +These commands are used to build the pacakge repository files. +""" +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import logging +import os +import pathlib +import shutil +import sys +from typing import TYPE_CHECKING + +from ptscripts import Context, command_group + +import tools.pkg +import tools.utils +from tools.utils import Version, get_salt_releases + +try: + import boto3 + from botocore.exceptions import ClientError +except ImportError: + print( + "\nPlease run 'python -m pip install -r " + "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info), + file=sys.stderr, + flush=True, + ) + raise + +log = logging.getLogger(__name__) + +# Define the command group +repo = command_group( + name="repo", + help="Packaging Repository Related Commands", + description=__doc__, + parent="pkg", +) + + +@repo.command(name="backup-previous-releases") +def backup_previous_releases(ctx: Context): + """ + Backup release bucket. + """ + _rclone(ctx, tools.utils.RELEASE_BUCKET_NAME, tools.utils.BACKUP_BUCKET_NAME) + ctx.info("Done") + + +@repo.command(name="restore-previous-releases") +def restore_previous_releases(ctx: Context): + """ + Restore release bucket from backup. + """ + _rclone(ctx, tools.utils.BACKUP_BUCKET_NAME, tools.utils.RELEASE_BUCKET_NAME) + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output is not None: + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"backup-complete=true\n") + ctx.info("Done") + + +def _rclone(ctx: Context, src: str, dst: str): + rclone = shutil.which("rclone") + if not rclone: + ctx.error("Could not find the rclone binary") + ctx.exit(1) + + if TYPE_CHECKING: + assert rclone + + env = os.environ.copy() + env["RCLONE_CONFIG_S3_TYPE"] = "s3" + cmdline: list[str] = [ + rclone, + "sync", + "--auto-confirm", + "--human-readable", + "--checksum", + "--color=always", + "--metadata", + "--s3-env-auth", + "--s3-location-constraint=us-west-2", + "--s3-provider=AWS", + "--s3-region=us-west-2", + "--stats-file-name-length=0", + "--stats-one-line", + "--stats=5s", + "--transfers=50", + "--fast-list", + "--verbose", + ] + if src == tools.utils.RELEASE_BUCKET_NAME: + cmdline.append("--s3-storage-class=INTELLIGENT_TIERING") + cmdline.extend([f"s3://{src}", f"s3://{dst}"]) + ctx.info(f"Running: {' '.join(cmdline)}") + ret = ctx.run(*cmdline, env=env, check=False) + if ret.returncode: + ctx.error(f"Failed to sync from s3://{src} to s3://{dst}") + ctx.exit(1) + + +@repo.command( + name="confirm-unreleased", + arguments={ + "salt_version": { + "help": "The salt version to check", + }, + "repository": { + "help": ( + "The full repository name, ie, 'saltstack/salt' on GitHub " + "to run the checks against." + ) + }, + }, +) +def confirm_unreleased( + ctx: Context, salt_version: str, repository: str = "saltstack/salt" +): + """ + Confirm that the passed version is not yet tagged and/or released. + """ + releases = get_salt_releases(ctx, repository) + if Version(salt_version) in releases: + ctx.error(f"There's already a '{salt_version}' tag or github release.") + ctx.exit(1) + ctx.info(f"Could not find a release for Salt Version '{salt_version}'") + ctx.exit(0) + + +@repo.command( + name="confirm-staged", + arguments={ + "salt_version": { + "help": "The salt version to check", + }, + "repository": { + "help": ( + "The full repository name, ie, 'saltstack/salt' on GitHub " + "to run the checks against." + ) + }, + }, +) +def confirm_staged(ctx: Context, salt_version: str, repository: str = "saltstack/salt"): + """ + Confirm that the passed version has been staged for release. + """ + s3 = boto3.client("s3") + repo_release_files_path = pathlib.Path( + f"release-artifacts/{salt_version}/.release-files.json" + ) + repo_release_symlinks_path = pathlib.Path( + f"release-artifacts/{salt_version}/.release-symlinks.json" + ) + for remote_path in (repo_release_files_path, repo_release_symlinks_path): + try: + bucket_name = tools.utils.STAGING_BUCKET_NAME + ctx.info( + f"Checking for the presence of {remote_path} on bucket {bucket_name} ..." + ) + s3.head_object( + Bucket=bucket_name, + Key=str(remote_path), + ) + except ClientError as exc: + if "Error" not in exc.response: + log.exception(f"Could not get information about {remote_path}: {exc}") + ctx.exit(1) + if exc.response["Error"]["Code"] == "404": + ctx.error(f"Could not find {remote_path} in bucket.") + ctx.exit(1) + if exc.response["Error"]["Code"] == "400": + ctx.error(f"Could get information about {remote_path}: {exc}") + ctx.exit(1) + log.exception(f"Error getting information about {remote_path}: {exc}") + ctx.exit(1) + ctx.info(f"Version {salt_version} has been staged for release") + ctx.exit(0) diff --git a/tools/pkg/repo/create.py b/tools/pkg/repo/create.py new file mode 100644 index 00000000000..ec4b3331c42 --- /dev/null +++ b/tools/pkg/repo/create.py @@ -0,0 +1,1038 @@ +""" +These commands are used to build the pacakge repository files. +""" +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import hashlib +import json +import logging +import os +import pathlib +import shutil +import sys +import textwrap +from typing import TYPE_CHECKING + +from ptscripts import Context, command_group + +import tools.pkg +import tools.utils +from tools.utils import ( + Version, + create_full_repo_path, + create_top_level_repo_path, + get_repo_json_file_contents, + parse_versions, +) + +try: + import boto3 +except ImportError: + print( + "\nPlease run 'python -m pip install -r " + "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info), + file=sys.stderr, + flush=True, + ) + raise + +log = logging.getLogger(__name__) + +create = command_group( + name="create", + help="Packaging Repository Creation Related Commands", + parent=["pkg", "repo"], +) + + +_deb_distro_info = { + "debian": { + "10": { + "label": "deb10ary", + "codename": "buster", + "suitename": "oldstable", + }, + "11": { + "label": "deb11ary", + "codename": "bullseye", + "suitename": "stable", + }, + }, + "ubuntu": { + "20.04": { + "label": "salt_ubuntu2004", + "codename": "focal", + }, + "22.04": { + "label": "salt_ubuntu2204", + "codename": "jammy", + }, + }, +} + + +@create.command( + name="deb", + arguments={ + "salt_version": { + "help": ( + "The salt version for which to build the repository configuration files. " + "If not passed, it will be discovered by running 'python3 salt/version.py'." + ), + "required": True, + }, + "distro": { + "help": "The debian based distribution to build the repository for", + "choices": list(_deb_distro_info), + "required": True, + }, + "distro_version": { + "help": "The distro version.", + "required": True, + }, + "distro_arch": { + "help": "The distribution architecture", + "choices": ("x86_64", "amd64", "aarch64", "arm64"), + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + "nightly_build_from": { + "help": "Developement repository target", + }, + }, +) +def debian( + ctx: Context, + salt_version: str = None, + distro: str = None, + distro_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + distro_arch: str = "amd64", + nightly_build_from: str = None, +): + """ + Create the debian repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert distro is not None + assert distro_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + display_name = f"{distro.capitalize()} {distro_version}" + if distro_version not in _deb_distro_info[distro]: + ctx.error(f"Support for {display_name} is missing.") + ctx.exit(1) + + if distro_arch == "x86_64": + ctx.info(f"The {distro_arch} arch is an alias for 'amd64'. Adjusting.") + distro_arch = "amd64" + + if distro_arch == "aarch64": + ctx.info(f"The {distro_arch} arch is an alias for 'arm64'. Adjusting.") + distro_arch = "arm64" + + distro_details = _deb_distro_info[distro][distro_version] + + ctx.info("Distribution Details:") + ctx.info(distro_details) + if TYPE_CHECKING: + assert isinstance(distro_details["label"], str) + assert isinstance(distro_details["codename"], str) + assert isinstance(distro_details["suitename"], str) + label: str = distro_details["label"] + codename: str = distro_details["codename"] + + ftp_archive_config_suite = "" + if distro == "debian": + suitename: str = distro_details["suitename"] + ftp_archive_config_suite = ( + f"""\n APT::FTPArchive::Release::Suite "{suitename}";\n""" + ) + archive_description = f"SaltProject {display_name} Python 3{'' if not nightly_build_from else ' development'} Salt package repo" + ftp_archive_config = f"""\ + APT::FTPArchive::Release::Origin "SaltProject"; + APT::FTPArchive::Release::Label "{label}";{ftp_archive_config_suite} + APT::FTPArchive::Release::Codename "{codename}"; + APT::FTPArchive::Release::Architectures "{distro_arch}"; + APT::FTPArchive::Release::Components "main"; + APT::FTPArchive::Release::Description "{archive_description}"; + APT::FTPArchive::Release::Acquire-By-Hash "yes"; + Dir {{ + ArchiveDir "."; + }}; + BinDirectory "pool" {{ + Packages "dists/{codename}/main/binary-{distro_arch}/Packages"; + Sources "dists/{codename}/main/source/Sources"; + Contents "dists/{codename}/main/Contents-{distro_arch}"; + }} + """ + ctx.info("Creating repository directory structure ...") + create_repo_path = create_top_level_repo_path( + ctx, + repo_path, + salt_version, + distro, + distro_version=distro_version, + distro_arch=distro_arch, + nightly_build_from=nightly_build_from, + ) + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + + create_repo_path = create_full_repo_path( + ctx, + repo_path, + salt_version, + distro, + distro_version=distro_version, + distro_arch=distro_arch, + nightly_build_from=nightly_build_from, + ) + ftp_archive_config_file = create_repo_path / "apt-ftparchive.conf" + ctx.info(f"Writing {ftp_archive_config_file} ...") + ftp_archive_config_file.write_text(textwrap.dedent(ftp_archive_config)) + + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + + pool_path = create_repo_path / "pool" + pool_path.mkdir(exist_ok=True) + for fpath in incoming.iterdir(): + dpath = pool_path / fpath.name + ctx.info(f"Copying {fpath} to {dpath} ...") + shutil.copyfile(fpath, dpath) + if fpath.suffix == ".dsc": + ctx.info(f"Running 'debsign' on {dpath} ...") + ctx.run("debsign", "--re-sign", "-k", key_id, str(dpath), interactive=True) + + dists_path = create_repo_path / "dists" + symlink_parent_path = dists_path / codename / "main" + symlink_paths = ( + symlink_parent_path / "by-hash" / "SHA256", + symlink_parent_path / "source" / "by-hash" / "SHA256", + symlink_parent_path / f"binary-{distro_arch}" / "by-hash" / "SHA256", + ) + + for path in symlink_paths: + path.mkdir(exist_ok=True, parents=True) + + cmdline = ["apt-ftparchive", "generate", "apt-ftparchive.conf"] + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ctx.run(*cmdline, cwd=create_repo_path) + + ctx.info("Creating by-hash symlinks ...") + for path in symlink_paths: + for fpath in path.parent.parent.iterdir(): + if not fpath.is_file(): + continue + sha256sum = ctx.run("sha256sum", str(fpath), capture=True) + link = path / sha256sum.stdout.decode().split()[0] + link.symlink_to(f"../../{fpath.name}") + + cmdline = [ + "apt-ftparchive", + "--no-md5", + "--no-sha1", + "--no-sha512", + "release", + "-c", + "apt-ftparchive.conf", + f"dists/{codename}/", + ] + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ret = ctx.run(*cmdline, capture=True, cwd=create_repo_path) + release_file = dists_path / codename / "Release" + ctx.info(f"Writing {release_file} with the output of the previous command...") + release_file.write_bytes(ret.stdout) + + cmdline = [ + "gpg", + "-u", + key_id, + "-o", + f"dists/{codename}/InRelease", + "-a", + "-s", + "--clearsign", + f"dists/{codename}/Release", + ] + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ctx.run(*cmdline, cwd=create_repo_path) + + cmdline = [ + "gpg", + "-u", + key_id, + "-o", + f"dists/{codename}/Release.gpg", + "-a", + "-b", + "-s", + f"dists/{codename}/Release", + ] + + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ctx.run(*cmdline, cwd=create_repo_path) + if not nightly_build_from: + remote_versions = _get_remote_versions( + tools.utils.STAGING_BUCKET_NAME, + create_repo_path.parent.relative_to(repo_path), + ) + major_version = Version(salt_version).major + matching_major = None + for version in remote_versions: + if version.major == major_version: + matching_major = version + break + if not matching_major or matching_major <= salt_version: + major_link = create_repo_path.parent.parent / str(major_version) + ctx.info(f"Creating '{major_link.relative_to(repo_path)}' symlink ...") + major_link.symlink_to(f"minor/{salt_version}") + if not remote_versions or remote_versions[0] <= salt_version: + latest_link = create_repo_path.parent.parent / "latest" + ctx.info(f"Creating '{latest_link.relative_to(repo_path)}' symlink ...") + latest_link.symlink_to(f"minor/{salt_version}") + + ctx.info("Done") + + +_rpm_distro_info = { + "amazon": ["2"], + "redhat": ["7", "8", "9"], + "fedora": ["36", "37", "38"], + "photon": ["3", "4"], +} + + +@create.command( + name="rpm", + arguments={ + "salt_version": { + "help": ( + "The salt version for which to build the repository configuration files. " + "If not passed, it will be discovered by running 'python3 salt/version.py'." + ), + "required": True, + }, + "distro": { + "help": "The debian based distribution to build the repository for", + "choices": list(_rpm_distro_info), + "required": True, + }, + "distro_version": { + "help": "The distro version.", + "required": True, + }, + "distro_arch": { + "help": "The distribution architecture", + "choices": ("x86_64", "aarch64", "arm64"), + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + "nightly_build_from": { + "help": "Developement repository target", + }, + }, +) +def rpm( + ctx: Context, + salt_version: str = None, + distro: str = None, + distro_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + distro_arch: str = "amd64", + nightly_build_from: str = None, +): + """ + Create the redhat repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert distro is not None + assert distro_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + display_name = f"{distro.capitalize()} {distro_version}" + if distro_version not in _rpm_distro_info[distro]: + ctx.error(f"Support for {display_name} is missing.") + ctx.exit(1) + + if distro_arch == "aarch64": + ctx.info(f"The {distro_arch} arch is an alias for 'arm64'. Adjusting.") + distro_arch = "arm64" + + ctx.info("Creating repository directory structure ...") + create_repo_path = create_top_level_repo_path( + ctx, + repo_path, + salt_version, + distro, + distro_version=distro_version, + distro_arch=distro_arch, + nightly_build_from=nightly_build_from, + ) + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + + create_repo_path = create_full_repo_path( + ctx, + repo_path, + salt_version, + distro, + distro_version=distro_version, + distro_arch=distro_arch, + nightly_build_from=nightly_build_from, + ) + + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + + for fpath in incoming.iterdir(): + if ".src" in fpath.suffixes: + dpath = create_repo_path / "SRPMS" / fpath.name + else: + dpath = create_repo_path / fpath.name + ctx.info(f"Copying {fpath} to {dpath} ...") + shutil.copyfile(fpath, dpath) + if fpath.suffix == ".rpm": + ctx.info(f"Running 'rpmsign' on {dpath} ...") + ctx.run( + "rpmsign", + "--key-id", + key_id, + "--addsign", + "--digest-algo=sha256", + str(dpath), + ) + + createrepo = shutil.which("createrepo") + if createrepo is None: + container = "ghcr.io/saltstack/salt-ci-containers/packaging:centosstream-9" + ctx.info(f"Using docker container '{container}' to call 'createrepo'...") + uid = ctx.run("id", "-u", capture=True).stdout.strip().decode() + gid = ctx.run("id", "-g", capture=True).stdout.strip().decode() + ctx.run( + "docker", + "run", + "--rm", + "-v", + f"{create_repo_path.resolve()}:/code", + "-u", + f"{uid}:{gid}", + "-w", + "/code", + container, + "createrepo", + ".", + ) + else: + ctx.run("createrepo", ".", cwd=create_repo_path) + + if nightly_build_from: + repo_domain = os.environ.get("SALT_REPO_DOMAIN_RELEASE", "repo.saltproject.io") + else: + repo_domain = os.environ.get( + "SALT_REPO_DOMAIN_STAGING", "staging.repo.saltproject.io" + ) + + salt_repo_user = os.environ.get("SALT_REPO_USER") + if salt_repo_user: + log.info( + "SALT_REPO_USER: %s", + salt_repo_user[0] + "*" * (len(salt_repo_user) - 2) + salt_repo_user[-1], + ) + salt_repo_pass = os.environ.get("SALT_REPO_PASS") + if salt_repo_pass: + log.info( + "SALT_REPO_PASS: %s", + salt_repo_pass[0] + "*" * (len(salt_repo_pass) - 2) + salt_repo_pass[-1], + ) + if salt_repo_user and salt_repo_pass: + repo_domain = f"{salt_repo_user}:{salt_repo_pass}@{repo_domain}" + + def _create_repo_file(create_repo_path, url_suffix): + ctx.info(f"Creating '{repo_file_path.relative_to(repo_path)}' file ...") + if nightly_build_from: + base_url = f"salt-dev/{nightly_build_from}/" + repo_file_contents = "[salt-nightly-repo]" + elif "rc" in salt_version: + base_url = "salt_rc/" + repo_file_contents = "[salt-rc-repo]" + else: + base_url = "" + repo_file_contents = "[salt-repo]" + base_url += f"salt/py3/{distro}/{distro_version}/{distro_arch}/{url_suffix}" + if distro == "amazon": + distro_name = "Amazon Linux" + elif distro == "redhat": + distro_name = "RHEL/CentOS" + else: + distro_name = distro.capitalize() + + if distro != "photon" and int(distro_version) < 8: + failovermethod = "\n failovermethod=priority" + else: + failovermethod = "" + + repo_file_contents += textwrap.dedent( + f""" + name=Salt repo for {distro_name} {distro_version} PY3 + baseurl=https://{repo_domain}/{base_url} + skip_if_unavailable=True{failovermethod} + priority=10 + enabled=1 + enabled_metadata=1 + gpgcheck=1 + gpgkey=https://{repo_domain}/{base_url}/{tools.utils.GPG_KEY_FILENAME}.pub + """ + ) + create_repo_path.write_text(repo_file_contents) + + if nightly_build_from: + repo_file_path = create_repo_path.parent / "nightly.repo" + else: + repo_file_path = create_repo_path.parent / f"{create_repo_path.name}.repo" + + _create_repo_file(repo_file_path, f"minor/{salt_version}") + + if not nightly_build_from: + remote_versions = _get_remote_versions( + tools.utils.STAGING_BUCKET_NAME, + create_repo_path.parent.relative_to(repo_path), + ) + major_version = Version(salt_version).major + matching_major = None + for version in remote_versions: + if version.major == major_version: + matching_major = version + break + if not matching_major or matching_major <= salt_version: + major_link = create_repo_path.parent.parent / str(major_version) + ctx.info(f"Creating '{major_link.relative_to(repo_path)}' symlink ...") + major_link.symlink_to(f"minor/{salt_version}") + repo_file_path = create_repo_path.parent.parent / f"{major_version}.repo" + _create_repo_file(repo_file_path, str(major_version)) + if not remote_versions or remote_versions[0] <= salt_version: + latest_link = create_repo_path.parent.parent / "latest" + ctx.info(f"Creating '{latest_link.relative_to(repo_path)}' symlink ...") + latest_link.symlink_to(f"minor/{salt_version}") + repo_file_path = create_repo_path.parent.parent / "latest.repo" + _create_repo_file(repo_file_path, "latest") + + ctx.info("Done") + + +@create.command( + name="windows", + arguments={ + "salt_version": { + "help": "The salt version for which to build the repository", + "required": True, + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + "nightly_build_from": { + "help": "Developement repository target", + }, + }, +) +def windows( + ctx: Context, + salt_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + nightly_build_from: str = None, +): + """ + Create the windows repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + _create_onedir_based_repo( + ctx, + salt_version=salt_version, + nightly_build_from=nightly_build_from, + repo_path=repo_path, + incoming=incoming, + key_id=key_id, + distro="windows", + pkg_suffixes=(".msi", ".exe"), + ) + ctx.info("Done") + + +@create.command( + name="macos", + arguments={ + "salt_version": { + "help": "The salt version for which to build the repository", + "required": True, + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + "nightly_build_from": { + "help": "Developement repository target", + }, + }, +) +def macos( + ctx: Context, + salt_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + nightly_build_from: str = None, +): + """ + Create the windows repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + _create_onedir_based_repo( + ctx, + salt_version=salt_version, + nightly_build_from=nightly_build_from, + repo_path=repo_path, + incoming=incoming, + key_id=key_id, + distro="macos", + pkg_suffixes=(".pkg",), + ) + ctx.info("Done") + + +@create.command( + name="onedir", + arguments={ + "salt_version": { + "help": "The salt version for which to build the repository", + "required": True, + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + "nightly_build_from": { + "help": "Developement repository target", + }, + }, +) +def onedir( + ctx: Context, + salt_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + nightly_build_from: str = None, +): + """ + Create the onedir repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + _create_onedir_based_repo( + ctx, + salt_version=salt_version, + nightly_build_from=nightly_build_from, + repo_path=repo_path, + incoming=incoming, + key_id=key_id, + distro="onedir", + pkg_suffixes=(".xz", ".zip"), + ) + ctx.info("Done") + + +@create.command( + name="src", + arguments={ + "salt_version": { + "help": "The salt version for which to build the repository", + "required": True, + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + "nightly_build_from": { + "help": "Developement repository target", + }, + }, +) +def src( + ctx: Context, + salt_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + nightly_build_from: str = None, +): + """ + Create the onedir repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + + ctx.info("Creating repository directory structure ...") + create_repo_path = create_top_level_repo_path( + ctx, + repo_path, + salt_version, + distro="src", + nightly_build_from=nightly_build_from, + ) + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + create_repo_path = create_repo_path / salt_version + create_repo_path.mkdir(exist_ok=True, parents=True) + hashes_base_path = create_repo_path / f"salt-{salt_version}" + for fpath in incoming.iterdir(): + if fpath.suffix not in (".gz",): + continue + ctx.info(f"* Processing {fpath} ...") + dpath = create_repo_path / fpath.name + ctx.info(f"Copying {fpath} to {dpath} ...") + shutil.copyfile(fpath, dpath) + for hash_name in ("blake2b", "sha512", "sha3_512"): + ctx.info(f" * Calculating {hash_name} ...") + hexdigest = _get_file_checksum(fpath, hash_name) + with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh: + wfh.write(f"{hexdigest} {dpath.name}\n") + with open(f"{dpath}.{hash_name}", "a+") as wfh: + wfh.write(f"{hexdigest} {dpath.name}\n") + + for fpath in create_repo_path.iterdir(): + if fpath.suffix in (".pub", ".gpg"): + continue + tools.utils.gpg_sign(ctx, key_id, fpath) + + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + ctx.info("Done") + + +def _get_remote_versions(bucket_name: str, remote_path: str): + log.info( + "Getting remote versions from bucket %r under path: %s", + bucket_name, + remote_path, + ) + remote_path = str(remote_path) + if not remote_path.endswith("/"): + remote_path += "/" + + s3 = boto3.client("s3") + ret = s3.list_objects( + Bucket=bucket_name, + Delimiter="/", + Prefix=remote_path, + ) + if "CommonPrefixes" not in ret: + return [] + versions = [] + for entry in ret["CommonPrefixes"]: + _, version = entry["Prefix"].rstrip("/").rsplit("/", 1) + if version == "latest": + continue + versions.append(Version(version)) + versions.sort(reverse=True) + log.info("Remote versions collected: %s", versions) + return versions + + +def _create_onedir_based_repo( + ctx: Context, + salt_version: str, + nightly_build_from: str | None, + repo_path: pathlib.Path, + incoming: pathlib.Path, + key_id: str, + distro: str, + pkg_suffixes: tuple[str, ...], +): + ctx.info("Creating repository directory structure ...") + create_repo_path = create_top_level_repo_path( + ctx, + repo_path, + salt_version, + distro, + nightly_build_from=nightly_build_from, + ) + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + + create_repo_path = create_full_repo_path( + ctx, + repo_path, + salt_version, + distro, + nightly_build_from=nightly_build_from, + ) + if not nightly_build_from: + repo_json_path = create_repo_path.parent.parent / "repo.json" + else: + repo_json_path = create_repo_path.parent / "repo.json" + + if nightly_build_from: + bucket_name = tools.utils.RELEASE_BUCKET_NAME + else: + bucket_name = tools.utils.STAGING_BUCKET_NAME + + release_json = {} + + copy_exclusions = ( + ".blake2b", + ".sha512", + ".sha3_512", + ".BLAKE2B", + ".SHA512", + ".SHA3_512", + ".json", + ) + hashes_base_path = create_repo_path / f"salt-{salt_version}" + for fpath in incoming.iterdir(): + if fpath.suffix in copy_exclusions: + continue + ctx.info(f"* Processing {fpath} ...") + dpath = create_repo_path / fpath.name + ctx.info(f"Copying {fpath} to {dpath} ...") + shutil.copyfile(fpath, dpath) + if "-amd64" in dpath.name.lower(): + arch = "amd64" + elif "-x86_64" in dpath.name.lower(): + arch = "x86_64" + elif "-x86" in dpath.name.lower(): + arch = "x86" + elif "-aarch64" in dpath.name.lower(): + arch = "aarch64" + else: + ctx.error( + f"Cannot pickup the right architecture from the filename '{dpath.name}'." + ) + ctx.exit(1) + if distro == "onedir": + if "-onedir-linux-" in dpath.name.lower(): + release_os = "linux" + elif "-onedir-darwin-" in dpath.name.lower(): + release_os = "macos" + elif "-onedir-windows-" in dpath.name.lower(): + release_os = "windows" + else: + ctx.error( + f"Cannot pickup the right OS from the filename '{dpath.name}'." + ) + ctx.exit(1) + else: + release_os = distro + release_json[dpath.name] = { + "name": dpath.name, + "version": salt_version, + "os": release_os, + "arch": arch, + } + for hash_name in ("blake2b", "sha512", "sha3_512"): + ctx.info(f" * Calculating {hash_name} ...") + hexdigest = _get_file_checksum(fpath, hash_name) + release_json[dpath.name][hash_name.upper()] = hexdigest + with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh: + wfh.write(f"{hexdigest} {dpath.name}\n") + with open(f"{dpath}.{hash_name}", "a+") as wfh: + wfh.write(f"{hexdigest} {dpath.name}\n") + + for fpath in create_repo_path.iterdir(): + if fpath.suffix in pkg_suffixes: + continue + tools.utils.gpg_sign(ctx, key_id, fpath) + + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + + repo_json = get_repo_json_file_contents( + ctx, bucket_name=bucket_name, repo_path=repo_path, repo_json_path=repo_json_path + ) + if nightly_build_from: + ctx.info(f"Writing {repo_json_path} ...") + repo_json_path.write_text(json.dumps(repo_json, sort_keys=True)) + return + + major_version = Version(salt_version).major + minor_repo_json_path = create_repo_path.parent / "repo.json" + minor_repo_json = get_repo_json_file_contents( + ctx, + bucket_name=bucket_name, + repo_path=repo_path, + repo_json_path=minor_repo_json_path, + ) + minor_repo_json[salt_version] = release_json + versions = parse_versions(*list(minor_repo_json)) + ctx.info( + f"Collected versions from {minor_repo_json_path.relative_to(repo_path)}: " + f"{', '.join(str(vs) for vs in versions)}" + ) + minor_versions = [v for v in versions if v.major == major_version] + ctx.info( + f"Collected versions(Matching major: {major_version}) from " + f"{minor_repo_json_path.relative_to(repo_path)}: " + f"{', '.join(str(vs) for vs in minor_versions)}" + ) + if not versions: + latest_version = Version(salt_version) + else: + latest_version = versions[0] + if not minor_versions: + latest_minor_version = Version(salt_version) + else: + latest_minor_version = minor_versions[0] + + ctx.info(f"Release Version: {salt_version}") + ctx.info(f"Latest Repo Version: {latest_version}") + ctx.info(f"Latest Release Minor Version: {latest_minor_version}") + + latest_link = create_repo_path.parent.parent / "latest" + if latest_version <= salt_version: + repo_json["latest"] = release_json + ctx.info(f"Creating '{latest_link.relative_to(repo_path)}' symlink ...") + if latest_link.exists(): + latest_link.unlink() + latest_link.symlink_to(f"minor/{salt_version}") + else: + ctx.info( + f"Not creating the '{latest_link.relative_to(repo_path)}' symlink " + f"since {latest_version} > {salt_version}" + ) + + major_link = create_repo_path.parent.parent / str(major_version) + if latest_minor_version <= salt_version: + minor_repo_json["latest"] = release_json + # This is the latest minor, update the major in the top level repo.json + # to this version + repo_json[str(major_version)] = release_json + ctx.info(f"Creating '{major_link.relative_to(repo_path)}' symlink ...") + if major_link.exists(): + major_link.unlink() + major_link.symlink_to(f"minor/{salt_version}") + else: + ctx.info( + f"Not creating the '{major_link.relative_to(repo_path)}' symlink " + f"since {latest_minor_version} > {salt_version}" + ) + + ctx.info(f"Writing {minor_repo_json_path} ...") + minor_repo_json_path.write_text(json.dumps(minor_repo_json, sort_keys=True)) + + ctx.info(f"Writing {repo_json_path} ...") + repo_json_path.write_text(json.dumps(repo_json, sort_keys=True)) + + +def _get_file_checksum(fpath: pathlib.Path, hash_name: str) -> str: + + with fpath.open("rb") as rfh: + try: + digest = hashlib.file_digest(rfh, hash_name) # type: ignore[attr-defined] + except AttributeError: + # Python < 3.11 + buf = bytearray(2**18) # Reusable buffer to reduce allocations. + view = memoryview(buf) + digest = getattr(hashlib, hash_name)() + while True: + size = rfh.readinto(buf) + if size == 0: + break # EOF + digest.update(view[:size]) + hexdigest: str = digest.hexdigest() + return hexdigest diff --git a/tools/pkg/repo/publish.py b/tools/pkg/repo/publish.py new file mode 100644 index 00000000000..cc6a92235c4 --- /dev/null +++ b/tools/pkg/repo/publish.py @@ -0,0 +1,653 @@ +""" +These commands are used to build the pacakge repository files. +""" +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import fnmatch +import json +import logging +import os +import pathlib +import re +import sys +import tempfile +import textwrap +from typing import TYPE_CHECKING, Any + +import packaging.version +from ptscripts import Context, command_group + +import tools.pkg +import tools.utils +from tools.utils import ( + Version, + create_full_repo_path, + get_repo_json_file_contents, + get_salt_releases, + parse_versions, +) + +try: + import boto3 + from botocore.exceptions import ClientError +except ImportError: + print( + "\nPlease run 'python -m pip install -r " + "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info), + file=sys.stderr, + flush=True, + ) + raise + +log = logging.getLogger(__name__) + +publish = command_group( + name="publish", + help="Packaging Repository Publication Related Commands", + parent=["pkg", "repo"], +) + + +@publish.command( + arguments={ + "repo_path": { + "help": "Local path for the repository that shall be published.", + }, + "salt_version": { + "help": "The salt version for which to build the repository", + "required": True, + }, + } +) +def nightly(ctx: Context, repo_path: pathlib.Path, salt_version: str = None): + """ + Publish to the nightly bucket. + """ + if TYPE_CHECKING: + assert salt_version is not None + _publish_repo( + ctx, repo_path=repo_path, nightly_build=True, salt_version=salt_version + ) + + +@publish.command( + arguments={ + "repo_path": { + "help": "Local path for the repository that shall be published.", + }, + "salt_version": { + "help": "The salt version for which to build the repository", + "required": True, + }, + } +) +def staging(ctx: Context, repo_path: pathlib.Path, salt_version: str = None): + """ + Publish to the staging bucket. + """ + if TYPE_CHECKING: + assert salt_version is not None + _publish_repo(ctx, repo_path=repo_path, stage=True, salt_version=salt_version) + + +@publish.command( + arguments={ + "salt_version": { + "help": "The salt version to release.", + }, + } +) +def release(ctx: Context, salt_version: str): + """ + Publish to the release bucket. + """ + if "rc" in salt_version: + bucket_folder = "salt_rc/salt/py3" + else: + bucket_folder = "salt/py3" + + files_to_copy: list[str] + directories_to_delete: list[str] = [] + + ctx.info("Grabbing remote file listing of files to copy...") + s3 = boto3.client("s3") + repo_release_files_path = pathlib.Path( + f"release-artifacts/{salt_version}/.release-files.json" + ) + repo_release_symlinks_path = pathlib.Path( + f"release-artifacts/{salt_version}/.release-symlinks.json" + ) + with tempfile.TemporaryDirectory(prefix=f"{salt_version}_release_") as tsd: + local_release_files_path = pathlib.Path(tsd) / repo_release_files_path.name + try: + bucket_name = tools.utils.STAGING_BUCKET_NAME + with local_release_files_path.open("wb") as wfh: + ctx.info( + f"Downloading {repo_release_files_path} from bucket {bucket_name} ..." + ) + s3.download_fileobj( + Bucket=bucket_name, + Key=str(repo_release_files_path), + Fileobj=wfh, + ) + files_to_copy = json.loads(local_release_files_path.read_text()) + except ClientError as exc: + if "Error" not in exc.response: + log.exception(f"Error downloading {repo_release_files_path}: {exc}") + ctx.exit(1) + if exc.response["Error"]["Code"] == "404": + ctx.error(f"Could not find {repo_release_files_path} in bucket.") + ctx.exit(1) + if exc.response["Error"]["Code"] == "400": + ctx.error( + f"Could not download {repo_release_files_path} from bucket: {exc}" + ) + ctx.exit(1) + log.exception(f"Error downloading {repo_release_files_path}: {exc}") + ctx.exit(1) + local_release_symlinks_path = ( + pathlib.Path(tsd) / repo_release_symlinks_path.name + ) + try: + with local_release_symlinks_path.open("wb") as wfh: + ctx.info( + f"Downloading {repo_release_symlinks_path} from bucket {bucket_name} ..." + ) + s3.download_fileobj( + Bucket=bucket_name, + Key=str(repo_release_symlinks_path), + Fileobj=wfh, + ) + directories_to_delete = json.loads(local_release_symlinks_path.read_text()) + except ClientError as exc: + if "Error" not in exc.response: + log.exception(f"Error downloading {repo_release_symlinks_path}: {exc}") + ctx.exit(1) + if exc.response["Error"]["Code"] == "404": + ctx.error(f"Could not find {repo_release_symlinks_path} in bucket.") + ctx.exit(1) + if exc.response["Error"]["Code"] == "400": + ctx.error( + f"Could not download {repo_release_symlinks_path} from bucket: {exc}" + ) + ctx.exit(1) + log.exception(f"Error downloading {repo_release_symlinks_path}: {exc}") + ctx.exit(1) + + if directories_to_delete: + with tools.utils.create_progress_bar() as progress: + task = progress.add_task( + "Deleting directories to override.", + total=len(directories_to_delete), + ) + for directory in directories_to_delete: + try: + objects_to_delete: list[dict[str, str]] = [] + for path in _get_repo_file_list( + bucket_name=tools.utils.RELEASE_BUCKET_NAME, + bucket_folder=bucket_folder, + glob_match=f"{directory}/**", + ): + objects_to_delete.append({"Key": path}) + if objects_to_delete: + s3.delete_objects( + Bucket=tools.utils.RELEASE_BUCKET_NAME, + Delete={"Objects": objects_to_delete}, + ) + except ClientError: + log.exception("Failed to delete remote files") + finally: + progress.update(task, advance=1) + + already_copied_files: list[str] = [] + s3 = boto3.client("s3") + dot_repo_files = [] + with tools.utils.create_progress_bar() as progress: + task = progress.add_task( + "Copying files between buckets", total=len(files_to_copy) + ) + for fpath in files_to_copy: + if fpath in already_copied_files: + continue + if fpath.endswith(".repo"): + dot_repo_files.append(fpath) + ctx.info(f" * Copying {fpath}") + try: + s3.copy_object( + Bucket=tools.utils.RELEASE_BUCKET_NAME, + Key=fpath, + CopySource={ + "Bucket": tools.utils.STAGING_BUCKET_NAME, + "Key": fpath, + }, + MetadataDirective="COPY", + TaggingDirective="COPY", + ServerSideEncryption="AES256", + ) + already_copied_files.append(fpath) + except ClientError: + log.exception(f"Failed to copy {fpath}") + finally: + progress.update(task, advance=1) + + # Now let's get the onedir based repositories where we need to update several repo.json + major_version = packaging.version.parse(salt_version).major + with tempfile.TemporaryDirectory(prefix=f"{salt_version}_release_") as tsd: + repo_path = pathlib.Path(tsd) + for distro in ("windows", "macos", "onedir"): + + create_repo_path = create_full_repo_path( + ctx, + repo_path, + salt_version, + distro=distro, + ) + repo_json_path = create_repo_path.parent.parent / "repo.json" + + release_repo_json = get_repo_json_file_contents( + ctx, + bucket_name=tools.utils.RELEASE_BUCKET_NAME, + repo_path=repo_path, + repo_json_path=repo_json_path, + ) + minor_repo_json_path = create_repo_path.parent / "repo.json" + + staging_minor_repo_json = get_repo_json_file_contents( + ctx, + bucket_name=tools.utils.STAGING_BUCKET_NAME, + repo_path=repo_path, + repo_json_path=minor_repo_json_path, + ) + release_minor_repo_json = get_repo_json_file_contents( + ctx, + bucket_name=tools.utils.RELEASE_BUCKET_NAME, + repo_path=repo_path, + repo_json_path=minor_repo_json_path, + ) + + release_json = staging_minor_repo_json[salt_version] + + major_version = Version(salt_version).major + versions = parse_versions(*list(release_minor_repo_json)) + ctx.info( + f"Collected versions from {minor_repo_json_path.relative_to(repo_path)}: " + f"{', '.join(str(vs) for vs in versions)}" + ) + minor_versions = [v for v in versions if v.major == major_version] + ctx.info( + f"Collected versions(Matching major: {major_version}) from " + f"{minor_repo_json_path.relative_to(repo_path)}: " + f"{', '.join(str(vs) for vs in minor_versions)}" + ) + if not versions: + latest_version = Version(salt_version) + else: + latest_version = versions[0] + if not minor_versions: + latest_minor_version = Version(salt_version) + else: + latest_minor_version = minor_versions[0] + + ctx.info(f"Release Version: {salt_version}") + ctx.info(f"Latest Repo Version: {latest_version}") + ctx.info(f"Latest Release Minor Version: {latest_minor_version}") + + # Add the minor version + release_minor_repo_json[salt_version] = release_json + + if latest_version <= salt_version: + release_repo_json["latest"] = release_json + + if latest_minor_version <= salt_version: + release_minor_repo_json["latest"] = release_json + + ctx.info(f"Writing {minor_repo_json_path} ...") + minor_repo_json_path.write_text( + json.dumps(release_minor_repo_json, sort_keys=True) + ) + ctx.info(f"Writing {repo_json_path} ...") + repo_json_path.write_text(json.dumps(release_repo_json, sort_keys=True)) + + # And now, let's get the several rpm "*.repo" files to update the base + # domain from staging to release + release_domain = os.environ.get( + "SALT_REPO_DOMAIN_RELEASE", "repo.saltproject.io" + ) + for path in dot_repo_files: + repo_file_path = repo_path.joinpath(path) + repo_file_path.parent.mkdir(exist_ok=True, parents=True) + bucket_name = tools.utils.STAGING_BUCKET_NAME + try: + ret = s3.head_object(Bucket=bucket_name, Key=path) + ctx.info( + f"Downloading existing '{repo_file_path.relative_to(repo_path)}' " + f"file from bucket {bucket_name}" + ) + size = ret["ContentLength"] + with repo_file_path.open("wb") as wfh: + with tools.utils.create_progress_bar( + file_progress=True + ) as progress: + task = progress.add_task( + description="Downloading...", total=size + ) + s3.download_fileobj( + Bucket=bucket_name, + Key=path, + Fileobj=wfh, + Callback=tools.utils.UpdateProgress(progress, task), + ) + updated_contents = re.sub( + r"^(baseurl|gpgkey)=https://([^/]+)/(.*)$", + rf"\1=https://{release_domain}/\3", + repo_file_path.read_text(), + flags=re.MULTILINE, + ) + ctx.info(f"Updated '{repo_file_path.relative_to(repo_path)}:") + ctx.print(updated_contents) + repo_file_path.write_text(updated_contents) + except ClientError as exc: + if "Error" not in exc.response: + raise + if exc.response["Error"]["Code"] != "404": + raise + ctx.info(f"Could not find {repo_file_path} in bucket {bucket_name}") + + for dirpath, dirnames, filenames in os.walk(repo_path, followlinks=True): + for path in filenames: + upload_path = pathlib.Path(dirpath, path) + relpath = upload_path.relative_to(repo_path) + size = upload_path.stat().st_size + ctx.info(f" {relpath}") + with tools.utils.create_progress_bar(file_progress=True) as progress: + task = progress.add_task(description="Uploading...", total=size) + s3.upload_file( + str(upload_path), + tools.utils.RELEASE_BUCKET_NAME, + str(relpath), + Callback=tools.utils.UpdateProgress(progress, task), + ) + + +@publish.command( + arguments={ + "salt_version": { + "help": "The salt version to release.", + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "repository": { + "help": ( + "The full repository name, ie, 'saltstack/salt' on GitHub " + "to run the checks against." + ) + }, + } +) +def github( + ctx: Context, + salt_version: str, + key_id: str = None, + repository: str = "saltstack/salt", +): + """ + Publish the release on GitHub releases. + """ + if TYPE_CHECKING: + assert key_id is not None + + s3 = boto3.client("s3") + + # Let's download the release artifacts stored in staging + artifacts_path = pathlib.Path.cwd() / "release-artifacts" + artifacts_path.mkdir(exist_ok=True) + release_artifacts_listing: dict[pathlib.Path, int] = {} + continuation_token = None + while True: + kwargs: dict[str, str] = {} + if continuation_token: + kwargs["ContinuationToken"] = continuation_token + ret = s3.list_objects_v2( + Bucket=tools.utils.STAGING_BUCKET_NAME, + Prefix=f"release-artifacts/{salt_version}", + FetchOwner=False, + **kwargs, + ) + contents = ret.pop("Contents", None) + if contents is None: + break + for entry in contents: + entry_path = pathlib.Path(entry["Key"]) + if entry_path.name.startswith("."): + continue + release_artifacts_listing[entry_path] = entry["Size"] + if not ret["IsTruncated"]: + break + continuation_token = ret["NextContinuationToken"] + + for entry_path, size in release_artifacts_listing.items(): + ctx.info(f" * {entry_path.name}") + local_path = artifacts_path / entry_path.name + with local_path.open("wb") as wfh: + with tools.utils.create_progress_bar(file_progress=True) as progress: + task = progress.add_task(description="Downloading...", total=size) + s3.download_fileobj( + Bucket=tools.utils.STAGING_BUCKET_NAME, + Key=str(entry_path), + Fileobj=wfh, + Callback=tools.utils.UpdateProgress(progress, task), + ) + + for artifact in artifacts_path.iterdir(): + if artifact.suffix in (".patch", ".asc", ".gpg", ".pub"): + continue + tools.utils.gpg_sign(ctx, key_id, artifact) + + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, artifacts_path) + + release_message = f"""\ + # Welcome to Salt v{salt_version} + + | :exclamation: ATTENTION | + |:-------------------------------------------------------------------------------------------------------------------------| + | The archives generated by GitHub(`Source code(zip)`, `Source code(tar.gz)`) will not report Salt's version properly. | + | Please use the tarball generated by The Salt Project Team(`salt-{salt_version}.tar.gz`). + """ + release_message_path = artifacts_path / "gh-release-body.md" + release_message_path.write_text(textwrap.dedent(release_message).strip()) + + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output is None: + ctx.warn("The 'GITHUB_OUTPUT' variable is not set. Stop processing.") + ctx.exit(0) + + if TYPE_CHECKING: + assert github_output is not None + + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"release-messsage-file={release_message_path.resolve()}\n") + + releases = get_salt_releases(ctx, repository) + if Version(salt_version) >= releases[-1]: + make_latest = True + else: + make_latest = False + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"make-latest={json.dumps(make_latest)}\n") + + artifacts_to_upload = [] + for artifact in artifacts_path.iterdir(): + if artifact.suffix == ".patch": + continue + if artifact.name == release_message_path.name: + continue + artifacts_to_upload.append(str(artifact.resolve())) + + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"release-artifacts={','.join(artifacts_to_upload)}\n") + ctx.exit(0) + + +def _get_repo_detailed_file_list( + bucket_name: str, + bucket_folder: str = "", + glob_match: str = "**", +) -> list[dict[str, Any]]: + s3 = boto3.client("s3") + listing: list[dict[str, Any]] = [] + continuation_token = None + while True: + kwargs: dict[str, str] = {} + if continuation_token: + kwargs["ContinuationToken"] = continuation_token + ret = s3.list_objects_v2( + Bucket=bucket_name, + Prefix=bucket_folder, + FetchOwner=False, + **kwargs, + ) + contents = ret.pop("Contents", None) + if contents is None: + break + for entry in contents: + if fnmatch.fnmatch(entry["Key"], glob_match): + listing.append(entry) + if not ret["IsTruncated"]: + break + continuation_token = ret["NextContinuationToken"] + return listing + + +def _get_repo_file_list( + bucket_name: str, bucket_folder: str, glob_match: str +) -> list[str]: + return [ + entry["Key"] + for entry in _get_repo_detailed_file_list( + bucket_name, bucket_folder, glob_match=glob_match + ) + ] + + +def _publish_repo( + ctx: Context, + repo_path: pathlib.Path, + salt_version: str, + nightly_build: bool = False, + stage: bool = False, +): + """ + Publish packaging repositories. + """ + if nightly_build: + bucket_name = tools.utils.RELEASE_BUCKET_NAME + elif stage: + bucket_name = tools.utils.STAGING_BUCKET_NAME + else: + bucket_name = tools.utils.RELEASE_BUCKET_NAME + + ctx.info("Preparing upload ...") + s3 = boto3.client("s3") + to_delete_paths: dict[pathlib.Path, list[dict[str, str]]] = {} + to_upload_paths: list[pathlib.Path] = [] + symlink_paths: list[str] = [] + uploaded_files: list[str] = [] + for dirpath, dirnames, filenames in os.walk(repo_path, followlinks=True): + for dirname in dirnames: + path = pathlib.Path(dirpath, dirname) + if not path.is_symlink(): + continue + # This is a symlink, then we need to delete all files under + # that directory in S3 because S3 does not understand symlinks + # and we would end up adding files to that folder instead of + # replacing it. + try: + relpath = path.relative_to(repo_path) + ret = s3.list_objects( + Bucket=bucket_name, + Prefix=str(relpath), + ) + if "Contents" not in ret: + continue + objects = [] + for entry in ret["Contents"]: + objects.append({"Key": entry["Key"]}) + to_delete_paths[path] = objects + symlink_paths.append(str(relpath)) + except ClientError as exc: + if "Error" not in exc.response: + raise + if exc.response["Error"]["Code"] != "404": + raise + + for fpath in filenames: + path = pathlib.Path(dirpath, fpath) + to_upload_paths.append(path) + + with tools.utils.create_progress_bar() as progress: + task = progress.add_task( + "Deleting directories to override.", total=len(to_delete_paths) + ) + for base, objects in to_delete_paths.items(): + relpath = base.relative_to(repo_path) + bucket_uri = f"s3://{bucket_name}/{relpath}" + progress.update(task, description=f"Deleting {bucket_uri}") + try: + ret = s3.delete_objects( + Bucket=bucket_name, + Delete={"Objects": objects}, + ) + except ClientError: + log.exception(f"Failed to delete {bucket_uri}") + finally: + progress.update(task, advance=1) + + try: + ctx.info("Uploading repository ...") + for upload_path in to_upload_paths: + relpath = upload_path.relative_to(repo_path) + size = upload_path.stat().st_size + ctx.info(f" {relpath}") + with tools.utils.create_progress_bar(file_progress=True) as progress: + task = progress.add_task(description="Uploading...", total=size) + s3.upload_file( + str(upload_path), + bucket_name, + str(relpath), + Callback=tools.utils.UpdateProgress(progress, task), + ExtraArgs={ + "Metadata": { + "x-amz-meta-salt-release-version": salt_version, + } + }, + ) + uploaded_files.append(str(relpath)) + if stage is True: + repo_files_path = f"release-artifacts/{salt_version}/.release-files.json" + ctx.info(f"Uploading {repo_files_path} ...") + s3.put_object( + Key=repo_files_path, + Bucket=bucket_name, + Body=json.dumps(uploaded_files).encode(), + Metadata={ + "x-amz-meta-salt-release-version": salt_version, + }, + ) + repo_symlinks_path = ( + f"release-artifacts/{salt_version}/.release-symlinks.json" + ) + ctx.info(f"Uploading {repo_symlinks_path} ...") + s3.put_object( + Key=repo_symlinks_path, + Bucket=bucket_name, + Body=json.dumps(symlink_paths).encode(), + Metadata={ + "x-amz-meta-salt-release-version": salt_version, + }, + ) + except KeyboardInterrupt: + pass diff --git a/tools/utils.py b/tools/utils.py index cb4379c61e0..28a79745844 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -1,8 +1,12 @@ # pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated from __future__ import annotations +import json import os import pathlib +import sys +from datetime import datetime +from typing import Any import packaging.version from ptscripts import Context @@ -16,6 +20,18 @@ from rich.progress import ( TransferSpeedColumn, ) +try: + import boto3 + from botocore.exceptions import ClientError +except ImportError: + print( + "\nPlease run 'python -m pip install -r " + "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info), + file=sys.stderr, + flush=True, + ) + raise + REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent GPG_KEY_FILENAME = "SALT-PROJECT-GPG-PUBKEY-2023" SPB_ENVIRONMENT = os.environ.get("SPB_ENVIRONMENT") or "prod" @@ -169,3 +185,114 @@ def get_salt_releases(ctx: Context, repository: str) -> list[Version]: # We're not going to parse dash or docs releases versions.add(Version(name)) return sorted(versions) + + +def parse_versions(*versions: str) -> list[Version]: + _versions = [] + for version in set(versions): + if version == "latest": + continue + _versions.append(Version(version)) + if _versions: + _versions.sort(reverse=True) + return _versions + + +def get_repo_json_file_contents( + ctx: Context, + bucket_name: str, + repo_path: pathlib.Path, + repo_json_path: pathlib.Path, +) -> dict[str, Any]: + s3 = boto3.client("s3") + repo_json: dict[str, Any] = {} + try: + ret = s3.head_object( + Bucket=bucket_name, Key=str(repo_json_path.relative_to(repo_path)) + ) + ctx.info( + f"Downloading existing '{repo_json_path.relative_to(repo_path)}' file " + f"from bucket {bucket_name}" + ) + size = ret["ContentLength"] + with repo_json_path.open("wb") as wfh: + with create_progress_bar(file_progress=True) as progress: + task = progress.add_task(description="Downloading...", total=size) + s3.download_fileobj( + Bucket=bucket_name, + Key=str(repo_json_path.relative_to(repo_path)), + Fileobj=wfh, + Callback=UpdateProgress(progress, task), + ) + with repo_json_path.open() as rfh: + repo_json = json.load(rfh) + except ClientError as exc: + if "Error" not in exc.response: + raise + if exc.response["Error"]["Code"] != "404": + raise + ctx.info(f"Could not find {repo_json_path} in bucket {bucket_name}") + if repo_json: + ctx.print(repo_json, soft_wrap=True) + return repo_json + + +def create_top_level_repo_path( + ctx: Context, + repo_path: pathlib.Path, + salt_version: str, + distro: str, + distro_version: str | None = None, # pylint: disable=bad-whitespace + distro_arch: str | None = None, # pylint: disable=bad-whitespace + nightly_build_from: str | None = None, # pylint: disable=bad-whitespace +): + create_repo_path = repo_path + if nightly_build_from: + create_repo_path = ( + create_repo_path + / "salt-dev" + / nightly_build_from + / datetime.utcnow().strftime("%Y-%m-%d") + ) + create_repo_path.mkdir(exist_ok=True, parents=True) + with ctx.chdir(create_repo_path.parent): + latest_nightly_symlink = pathlib.Path("latest") + if not latest_nightly_symlink.exists(): + ctx.info( + f"Creating 'latest' symlink to '{create_repo_path.relative_to(repo_path)}' ..." + ) + latest_nightly_symlink.symlink_to( + create_repo_path.name, target_is_directory=True + ) + elif "rc" in salt_version: + create_repo_path = create_repo_path / "salt_rc" + create_repo_path = create_repo_path / "salt" / "py3" / distro + if distro_version: + create_repo_path = create_repo_path / distro_version + if distro_arch: + create_repo_path = create_repo_path / distro_arch + create_repo_path.mkdir(exist_ok=True, parents=True) + return create_repo_path + + +def create_full_repo_path( + ctx: Context, + repo_path: pathlib.Path, + salt_version: str, + distro: str, + distro_version: str | None = None, # pylint: disable=bad-whitespace + distro_arch: str | None = None, # pylint: disable=bad-whitespace + nightly_build_from: str | None = None, # pylint: disable=bad-whitespace +): + create_repo_path = create_top_level_repo_path( + ctx, + repo_path, + salt_version, + distro, + distro_version, + distro_arch, + nightly_build_from=nightly_build_from, + ) + create_repo_path = create_repo_path / "minor" / salt_version + create_repo_path.mkdir(exist_ok=True, parents=True) + return create_repo_path From dcfdc6b216d20a7221db733750dc4e228ff9bebc Mon Sep 17 00:00:00 2001 From: MKLeb Date: Mon, 8 May 2023 15:30:25 -0400 Subject: [PATCH 003/152] Address review comments (typos, docs) --- tools/pkg/repo/__init__.py | 2 +- tools/pkg/repo/create.py | 2 +- tools/pkg/repo/publish.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/pkg/repo/__init__.py b/tools/pkg/repo/__init__.py index 8a3cbd9c81f..d965fcfd923 100644 --- a/tools/pkg/repo/__init__.py +++ b/tools/pkg/repo/__init__.py @@ -1,5 +1,5 @@ """ -These commands are used to build the pacakge repository files. +These commands are used to build the package repository files. """ # pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated from __future__ import annotations diff --git a/tools/pkg/repo/create.py b/tools/pkg/repo/create.py index ec4b3331c42..60ed8ad0570 100644 --- a/tools/pkg/repo/create.py +++ b/tools/pkg/repo/create.py @@ -1,5 +1,5 @@ """ -These commands are used to build the pacakge repository files. +These commands are used to build the package repository files. """ # pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated from __future__ import annotations diff --git a/tools/pkg/repo/publish.py b/tools/pkg/repo/publish.py index cc6a92235c4..1c87d20b490 100644 --- a/tools/pkg/repo/publish.py +++ b/tools/pkg/repo/publish.py @@ -1,5 +1,5 @@ """ -These commands are used to build the pacakge repository files. +These commands are used to build the package repository files. """ # pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated from __future__ import annotations @@ -55,7 +55,7 @@ publish = command_group( "help": "Local path for the repository that shall be published.", }, "salt_version": { - "help": "The salt version for which to build the repository", + "help": "The salt version of the repository to publish", "required": True, }, } @@ -77,7 +77,7 @@ def nightly(ctx: Context, repo_path: pathlib.Path, salt_version: str = None): "help": "Local path for the repository that shall be published.", }, "salt_version": { - "help": "The salt version for which to build the repository", + "help": "The salt version of the repository to publish", "required": True, }, } From 99261e77a5448a8bddb64ca870703e14e6ed905f Mon Sep 17 00:00:00 2001 From: Frode Gundersen Date: Thu, 23 Feb 2023 22:49:52 +0000 Subject: [PATCH 004/152] migrate unit_states_test_linux_acl to pytest --- tests/pytests/unit/states/test_linux_acl.py | 539 ++++++++++++++++++ tests/unit/states/test_linux_acl.py | 589 -------------------- 2 files changed, 539 insertions(+), 589 deletions(-) create mode 100644 tests/pytests/unit/states/test_linux_acl.py delete mode 100644 tests/unit/states/test_linux_acl.py diff --git a/tests/pytests/unit/states/test_linux_acl.py b/tests/pytests/unit/states/test_linux_acl.py new file mode 100644 index 00000000000..976a57b8c4b --- /dev/null +++ b/tests/pytests/unit/states/test_linux_acl.py @@ -0,0 +1,539 @@ +""" + :codeauthor: Jayesh Kariya + + Test cases for salt.states.linux_acl +""" + +import pytest + +import salt.states.linux_acl as linux_acl +from salt.exceptions import CommandExecutionError +from tests.support.mock import MagicMock, patch + +pytestmark = [ + pytest.mark.skip_unless_on_linux( + reason="Only run on Linux", + ) +] + + +@pytest.fixture +def configure_loader_modules(): + return {linux_acl: {}} + + +def test_present(): + """ + Test to ensure a Linux ACL is present + """ + maxDiff = None + name = "/root" + acl_type = "users" + acl_name = "damian" + perms = "rwx" + + mock = MagicMock( + side_effect=[ + {name: {acl_type: [{acl_name: {"octal": 5}}]}}, + {name: {acl_type: [{acl_name: {"octal": 5}}]}}, + {name: {acl_type: [{acl_name: {"octal": 5}}]}}, + {name: {acl_type: [{}]}}, + {name: {acl_type: [{}]}}, + {name: {acl_type: [{}]}}, + { + name: {acl_type: [{acl_name: {"octal": 7}}]}, + name + "/foo": {acl_type: [{acl_name: {"octal": 5}}]}, + }, + { + name: {acl_type: [{acl_name: {"octal": 7}}]}, + name + "/foo": {acl_type: [{acl_name: {"octal": 7}}]}, + }, + {name: {acl_type: ""}}, + { + name: {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, + name + "/foo": {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, + }, + { + name: {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, + name + "/foo": {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, + }, + { + name: {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, + name + "/foo": {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, + }, + ] + ) + mock_modfacl = MagicMock(return_value=True) + + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Updated permissions will be applied for {}: r-x -> {}".format( + acl_name, perms + ) + ret = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": perms, + }, + "old": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": "r-x", + }, + }, + "result": None, + } + + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + # Update - test=False + with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Updated permissions for {}".format(acl_name) + ret = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": perms, + }, + "old": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": "r-x", + }, + }, + "result": True, + } + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + # Update - modfacl error + with patch.dict( + linux_acl.__salt__, + {"acl.modfacl": MagicMock(side_effect=CommandExecutionError("Custom err"))}, + ): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Error updating permissions for {}: Custom err".format(acl_name) + ret = { + "name": name, + "comment": comt, + "changes": {}, + "result": False, + } + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + # New - test=True + with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "New permissions will be applied for {}: {}".format( + acl_name, perms + ) + ret = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": perms, + } + }, + "result": None, + } + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + # New - test=False + with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Applied new permissions for {}".format(acl_name) + ret = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": perms, + } + }, + "result": True, + } + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + # New - modfacl error + with patch.dict( + linux_acl.__salt__, + {"acl.modfacl": MagicMock(side_effect=CommandExecutionError("Custom err"))}, + ): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Error updating permissions for {}: Custom err".format(acl_name) + ret = { + "name": name, + "comment": comt, + "changes": {}, + "result": False, + } + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + + # New - recurse true + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Updated permissions will be applied for {}: rwx -> {}".format( + acl_name, perms + ) + ret = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": perms, + }, + "old": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": "rwx", + }, + }, + "result": None, + } + + assert ( + linux_acl.present(name, acl_type, acl_name, perms, recurse=True) + == ret + ) + + # New - recurse true - nothing to do + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Permissions are in the desired state" + ret = {"name": name, "comment": comt, "changes": {}, "result": True} + + assert ( + linux_acl.present(name, acl_type, acl_name, perms, recurse=True) + == ret + ) + + # No acl type + comt = "ACL Type does not exist" + ret = {"name": name, "comment": comt, "result": False, "changes": {}} + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + + # default recurse false - nothing to do + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Permissions are in the desired state" + ret = {"name": name, "comment": comt, "changes": {}, "result": True} + + assert ( + linux_acl.present( + name, "d:" + acl_type, acl_name, perms, recurse=False + ) + == ret + ) + + # default recurse false - nothing to do + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Permissions are in the desired state" + ret = {"name": name, "comment": comt, "changes": {}, "result": True} + + assert ( + linux_acl.present( + name, "d:" + acl_type, acl_name, perms, recurse=False + ) + == ret + ) + + # default recurse true - nothing to do + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Permissions are in the desired state" + ret = {"name": name, "comment": comt, "changes": {}, "result": True} + + assert ( + linux_acl.present( + name, "d:" + acl_type, acl_name, perms, recurse=True + ) + == ret + ) + + +def test_absent(): + """ + Test to ensure a Linux ACL does not exist + """ + name = "/root" + acl_type = "users" + acl_name = "damian" + perms = "rwx" + + ret = {"name": name, "result": None, "comment": "", "changes": {}} + + mock = MagicMock( + side_effect=[ + {name: {acl_type: [{acl_name: {"octal": "A"}}]}}, + {name: {acl_type: ""}}, + ] + ) + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Removing permissions" + ret.update({"comment": comt}) + assert linux_acl.absent(name, acl_type, acl_name, perms) == ret + + comt = "ACL Type does not exist" + ret.update({"comment": comt, "result": False}) + assert linux_acl.absent(name, acl_type, acl_name, perms) == ret + + +def test_list_present(): + """ + Test to ensure a Linux ACL is present + """ + maxDiff = None + name = "/root" + acl_type = "user" + acl_names = ["root", "damian", "homer"] + acl_comment = {"owner": "root", "group": "root", "file": "/root"} + perms = "rwx" + + mock = MagicMock( + side_effect=[ + { + name: { + acl_type: [ + {acl_names[0]: {"octal": "A"}}, + {acl_names[1]: {"octal": "A"}}, + {acl_names[2]: {"octal": "A"}}, + ], + "comment": acl_comment, + } + }, + { + name: { + acl_type: [ + {acl_names[0]: {"octal": "A"}}, + {acl_names[1]: {"octal": "A"}}, + ], + "comment": acl_comment, + } + }, + { + name: { + acl_type: [ + {acl_names[0]: {"octal": "A"}}, + {acl_names[1]: {"octal": "A"}}, + ] + } + }, + {name: {acl_type: [{}]}}, + {name: {acl_type: [{}]}}, + {name: {acl_type: [{}]}}, + {name: {acl_type: ""}}, + ] + ) + mock_modfacl = MagicMock(return_value=True) + + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Updated permissions will be applied for {}: A -> {}".format( + acl_names, perms + ) + expected = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": ", ".join(acl_names), + "acl_type": acl_type, + "perms": 7, + }, + "old": { + "acl_name": ", ".join(acl_names), + "acl_type": acl_type, + "perms": "A", + }, + }, + "result": None, + } + + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert ret == expected + + # Update - test=False + with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Applied new permissions for {}".format(", ".join(acl_names)) + expected = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": ", ".join(acl_names), + "acl_type": acl_type, + "perms": "rwx", + } + }, + "result": True, + } + + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert expected == ret + + # Update - modfacl error + with patch.dict( + linux_acl.__salt__, + {"acl.modfacl": MagicMock(side_effect=CommandExecutionError("Custom err"))}, + ): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Error updating permissions for {}: Custom err".format(acl_names) + expected = { + "name": name, + "comment": comt, + "changes": {}, + "result": False, + } + + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert expected == ret + + # New - test=True + with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "New permissions will be applied for {}: {}".format( + acl_names, perms + ) + expected = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": ", ".join(acl_names), + "acl_type": acl_type, + "perms": perms, + } + }, + "result": None, + } + + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert expected == ret + + # New - test=False + with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Applied new permissions for {}".format(", ".join(acl_names)) + expected = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": ", ".join(acl_names), + "acl_type": acl_type, + "perms": perms, + } + }, + "result": True, + } + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert expected == ret + + # New - modfacl error + with patch.dict( + linux_acl.__salt__, + {"acl.modfacl": MagicMock(side_effect=CommandExecutionError("Custom err"))}, + ): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Error updating permissions for {}: Custom err".format(acl_names) + expected = { + "name": name, + "comment": comt, + "changes": {}, + "result": False, + } + + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert expected == ret + + # No acl type + comt = "ACL Type does not exist" + expected = { + "name": name, + "comment": comt, + "result": False, + "changes": {}, + } + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert expected == ret + + +def test_list_absent(): + """ + Test to ensure a Linux ACL does not exist + """ + name = "/root" + acl_type = "users" + acl_names = ["damian", "homer"] + perms = "rwx" + + ret = {"name": name, "result": None, "comment": "", "changes": {}} + + mock = MagicMock( + side_effect=[ + { + name: { + acl_type: [ + {acl_names[0]: {"octal": "A"}, acl_names[1]: {"octal": "A"}} + ] + } + }, + {name: {acl_type: ""}}, + ] + ) + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Removing permissions" + ret.update({"comment": comt}) + assert linux_acl.list_absent(name, acl_type, acl_names, perms) == ret + + comt = "ACL Type does not exist" + ret.update({"comment": comt, "result": False}) + assert linux_acl.list_absent(name, acl_type, acl_names) == ret + + +def test_absent_recursive(): + """ + Test to ensure a Linux ACL does not exist + """ + name = "/root" + acl_type = "users" + acl_name = "damian" + perms = "rwx" + + ret = {"name": name, "result": None, "comment": "", "changes": {}} + + mock = MagicMock( + side_effect=[ + { + name: {acl_type: [{acl_name: {"octal": 7}}]}, + name + "/foo": {acl_type: [{acl_name: {"octal": "A"}}]}, + } + ] + ) + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Removing permissions" + ret.update({"comment": comt}) + assert ( + linux_acl.absent(name, acl_type, acl_name, perms, recurse=True) == ret + ) diff --git a/tests/unit/states/test_linux_acl.py b/tests/unit/states/test_linux_acl.py deleted file mode 100644 index 2961fbad53a..00000000000 --- a/tests/unit/states/test_linux_acl.py +++ /dev/null @@ -1,589 +0,0 @@ -""" - :codeauthor: Jayesh Kariya -""" - -import pytest - -import salt.states.linux_acl as linux_acl -from salt.exceptions import CommandExecutionError -from tests.support.mixins import LoaderModuleMockMixin -from tests.support.mock import MagicMock, patch -from tests.support.unit import TestCase - - -@pytest.mark.skip_unless_on_linux -class LinuxAclTestCase(TestCase, LoaderModuleMockMixin): - """ - Test cases for salt.states.linux_acl - """ - - def setup_loader_modules(self): - return {linux_acl: {}} - - # 'present' function tests: 1 - - def test_present(self): - """ - Test to ensure a Linux ACL is present - """ - self.maxDiff = None - name = "/root" - acl_type = "users" - acl_name = "damian" - perms = "rwx" - - mock = MagicMock( - side_effect=[ - {name: {acl_type: [{acl_name: {"octal": 5}}]}}, - {name: {acl_type: [{acl_name: {"octal": 5}}]}}, - {name: {acl_type: [{acl_name: {"octal": 5}}]}}, - {name: {acl_type: [{}]}}, - {name: {acl_type: [{}]}}, - {name: {acl_type: [{}]}}, - { - name: {acl_type: [{acl_name: {"octal": 7}}]}, - name + "/foo": {acl_type: [{acl_name: {"octal": 5}}]}, - }, - { - name: {acl_type: [{acl_name: {"octal": 7}}]}, - name + "/foo": {acl_type: [{acl_name: {"octal": 7}}]}, - }, - {name: {acl_type: ""}}, - { - name: {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, - name + "/foo": {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, - }, - { - name: {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, - name + "/foo": {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, - }, - { - name: {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, - name + "/foo": {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, - }, - ] - ) - mock_modfacl = MagicMock(return_value=True) - - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Updated permissions will be applied for {}: r-x -> {}".format( - acl_name, perms - ) - ret = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": perms, - }, - "old": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": "r-x", - }, - }, - "result": None, - } - - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - # Update - test=False - with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Updated permissions for {}".format(acl_name) - ret = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": perms, - }, - "old": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": "r-x", - }, - }, - "result": True, - } - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - # Update - modfacl error - with patch.dict( - linux_acl.__salt__, - { - "acl.modfacl": MagicMock( - side_effect=CommandExecutionError("Custom err") - ) - }, - ): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Error updating permissions for {}: Custom err".format( - acl_name - ) - ret = { - "name": name, - "comment": comt, - "changes": {}, - "result": False, - } - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - # New - test=True - with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "New permissions will be applied for {}: {}".format( - acl_name, perms - ) - ret = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": perms, - } - }, - "result": None, - } - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - # New - test=False - with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Applied new permissions for {}".format(acl_name) - ret = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": perms, - } - }, - "result": True, - } - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - # New - modfacl error - with patch.dict( - linux_acl.__salt__, - { - "acl.modfacl": MagicMock( - side_effect=CommandExecutionError("Custom err") - ) - }, - ): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Error updating permissions for {}: Custom err".format( - acl_name - ) - ret = { - "name": name, - "comment": comt, - "changes": {}, - "result": False, - } - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - - # New - recurse true - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = ( - "Updated permissions will be applied for {}: rwx -> {}".format( - acl_name, perms - ) - ) - ret = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": perms, - }, - "old": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": "rwx", - }, - }, - "result": None, - } - - self.assertDictEqual( - linux_acl.present( - name, acl_type, acl_name, perms, recurse=True - ), - ret, - ) - - # New - recurse true - nothing to do - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Permissions are in the desired state" - ret = {"name": name, "comment": comt, "changes": {}, "result": True} - - self.assertDictEqual( - linux_acl.present( - name, acl_type, acl_name, perms, recurse=True - ), - ret, - ) - - # No acl type - comt = "ACL Type does not exist" - ret = {"name": name, "comment": comt, "result": False, "changes": {}} - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - - # default recurse false - nothing to do - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Permissions are in the desired state" - ret = {"name": name, "comment": comt, "changes": {}, "result": True} - - self.assertDictEqual( - linux_acl.present( - name, "d:" + acl_type, acl_name, perms, recurse=False - ), - ret, - ) - - # default recurse false - nothing to do - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Permissions are in the desired state" - ret = {"name": name, "comment": comt, "changes": {}, "result": True} - - self.assertDictEqual( - linux_acl.present( - name, "d:" + acl_type, acl_name, perms, recurse=False - ), - ret, - ) - - # default recurse true - nothing to do - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Permissions are in the desired state" - ret = {"name": name, "comment": comt, "changes": {}, "result": True} - - self.assertDictEqual( - linux_acl.present( - name, "d:" + acl_type, acl_name, perms, recurse=True - ), - ret, - ) - - # 'absent' function tests: 2 - - def test_absent(self): - """ - Test to ensure a Linux ACL does not exist - """ - name = "/root" - acl_type = "users" - acl_name = "damian" - perms = "rwx" - - ret = {"name": name, "result": None, "comment": "", "changes": {}} - - mock = MagicMock( - side_effect=[ - {name: {acl_type: [{acl_name: {"octal": "A"}}]}}, - {name: {acl_type: ""}}, - ] - ) - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Removing permissions" - ret.update({"comment": comt}) - self.assertDictEqual( - linux_acl.absent(name, acl_type, acl_name, perms), ret - ) - - comt = "ACL Type does not exist" - ret.update({"comment": comt, "result": False}) - self.assertDictEqual(linux_acl.absent(name, acl_type, acl_name, perms), ret) - - # 'list_present' function tests: 1 - - def test_list_present(self): - """ - Test to ensure a Linux ACL is present - """ - self.maxDiff = None - name = "/root" - acl_type = "user" - acl_names = ["root", "damian", "homer"] - acl_comment = {"owner": "root", "group": "root", "file": "/root"} - perms = "rwx" - - mock = MagicMock( - side_effect=[ - { - name: { - acl_type: [ - {acl_names[0]: {"octal": "A"}}, - {acl_names[1]: {"octal": "A"}}, - {acl_names[2]: {"octal": "A"}}, - ], - "comment": acl_comment, - } - }, - { - name: { - acl_type: [ - {acl_names[0]: {"octal": "A"}}, - {acl_names[1]: {"octal": "A"}}, - ], - "comment": acl_comment, - } - }, - { - name: { - acl_type: [ - {acl_names[0]: {"octal": "A"}}, - {acl_names[1]: {"octal": "A"}}, - ] - } - }, - {name: {acl_type: [{}]}}, - {name: {acl_type: [{}]}}, - {name: {acl_type: [{}]}}, - {name: {acl_type: ""}}, - ] - ) - mock_modfacl = MagicMock(return_value=True) - - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Updated permissions will be applied for {}: A -> {}".format( - acl_names, perms - ) - expected = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": ", ".join(acl_names), - "acl_type": acl_type, - "perms": 7, - }, - "old": { - "acl_name": ", ".join(acl_names), - "acl_type": acl_type, - "perms": "A", - }, - }, - "result": None, - } - - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(ret, expected) - - # Update - test=False - with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Applied new permissions for {}".format(", ".join(acl_names)) - expected = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": ", ".join(acl_names), - "acl_type": acl_type, - "perms": "rwx", - } - }, - "result": True, - } - - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(expected, ret) - - # Update - modfacl error - with patch.dict( - linux_acl.__salt__, - { - "acl.modfacl": MagicMock( - side_effect=CommandExecutionError("Custom err") - ) - }, - ): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Error updating permissions for {}: Custom err".format( - acl_names - ) - expected = { - "name": name, - "comment": comt, - "changes": {}, - "result": False, - } - - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(expected, ret) - - # New - test=True - with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "New permissions will be applied for {}: {}".format( - acl_names, perms - ) - expected = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": ", ".join(acl_names), - "acl_type": acl_type, - "perms": perms, - } - }, - "result": None, - } - - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(expected, ret) - - # New - test=False - with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Applied new permissions for {}".format(", ".join(acl_names)) - expected = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": ", ".join(acl_names), - "acl_type": acl_type, - "perms": perms, - } - }, - "result": True, - } - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(expected, ret) - - # New - modfacl error - with patch.dict( - linux_acl.__salt__, - { - "acl.modfacl": MagicMock( - side_effect=CommandExecutionError("Custom err") - ) - }, - ): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Error updating permissions for {}: Custom err".format( - acl_names - ) - expected = { - "name": name, - "comment": comt, - "changes": {}, - "result": False, - } - - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(expected, ret) - - # No acl type - comt = "ACL Type does not exist" - expected = { - "name": name, - "comment": comt, - "result": False, - "changes": {}, - } - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(expected, ret) - - # 'list_absent' function tests: 2 - - def test_list_absent(self): - """ - Test to ensure a Linux ACL does not exist - """ - name = "/root" - acl_type = "users" - acl_names = ["damian", "homer"] - perms = "rwx" - - ret = {"name": name, "result": None, "comment": "", "changes": {}} - - mock = MagicMock( - side_effect=[ - { - name: { - acl_type: [ - {acl_names[0]: {"octal": "A"}, acl_names[1]: {"octal": "A"}} - ] - } - }, - {name: {acl_type: ""}}, - ] - ) - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Removing permissions" - ret.update({"comment": comt}) - self.assertDictEqual( - linux_acl.list_absent(name, acl_type, acl_names, perms), ret - ) - - comt = "ACL Type does not exist" - ret.update({"comment": comt, "result": False}) - self.assertDictEqual(linux_acl.list_absent(name, acl_type, acl_names), ret) - - def test_absent_recursive(self): - """ - Test to ensure a Linux ACL does not exist - """ - name = "/root" - acl_type = "users" - acl_name = "damian" - perms = "rwx" - - ret = {"name": name, "result": None, "comment": "", "changes": {}} - - mock = MagicMock( - side_effect=[ - { - name: {acl_type: [{acl_name: {"octal": 7}}]}, - name + "/foo": {acl_type: [{acl_name: {"octal": "A"}}]}, - } - ] - ) - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Removing permissions" - ret.update({"comment": comt}) - self.assertDictEqual( - linux_acl.absent(name, acl_type, acl_name, perms, recurse=True), ret - ) From d629d474142bc7893d87014ad3dd26f104fec06b Mon Sep 17 00:00:00 2001 From: Frode Gundersen Date: Mon, 10 Apr 2023 11:38:53 -0600 Subject: [PATCH 005/152] Update tests/pytests/unit/states/test_linux_acl.py Co-authored-by: Pedro Algarvio --- tests/pytests/unit/states/test_linux_acl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pytests/unit/states/test_linux_acl.py b/tests/pytests/unit/states/test_linux_acl.py index 976a57b8c4b..60bbe55f51c 100644 --- a/tests/pytests/unit/states/test_linux_acl.py +++ b/tests/pytests/unit/states/test_linux_acl.py @@ -299,7 +299,6 @@ def test_list_present(): """ Test to ensure a Linux ACL is present """ - maxDiff = None name = "/root" acl_type = "user" acl_names = ["root", "damian", "homer"] From 816fdb8c206ad737c7201b9ae996f8b915065bc8 Mon Sep 17 00:00:00 2001 From: jeanluc Date: Sat, 6 May 2023 23:57:52 +0200 Subject: [PATCH 006/152] Add test for issue 64232 --- .../integration/modules/test_x509_v2.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/pytests/integration/modules/test_x509_v2.py b/tests/pytests/integration/modules/test_x509_v2.py index 2fd005778c5..99d0d213397 100644 --- a/tests/pytests/integration/modules/test_x509_v2.py +++ b/tests/pytests/integration/modules/test_x509_v2.py @@ -673,6 +673,35 @@ def test_sign_remote_certificate_copypath(x509_salt_call_cli, cert_args, tmp_pat assert (tmp_path / f"{cert.serial_number:x}.crt").exists() +def test_create_private_key(x509_salt_call_cli): + """ + Ensure calling from the CLI works as expected and does not complain + about unknown internal kwargs (__pub_fun etc). + """ + ret = x509_salt_call_cli.run("x509.create_private_key") + assert ret.returncode == 0 + assert ret.data + assert ret.data.startswith("-----BEGIN PRIVATE KEY-----") + + +def test_create_crl(x509_salt_call_cli, ca_key, ca_cert, x509_pkidir): + """ + Ensure calling from the CLI works as expected and does not complain + about unknown internal kwargs (__pub_fun etc). + """ + with pytest.helpers.temp_file("key", ca_key, x509_pkidir) as ca_keyfile: + with pytest.helpers.temp_file("cert", ca_cert, x509_pkidir) as ca_certfile: + ret = x509_salt_call_cli.run( + "x509.create_crl", + revoked=[], + signing_private_key=str(ca_keyfile), + signing_cert=str(ca_certfile), + ) + assert ret.returncode == 0 + assert ret.data + assert ret.data.startswith("-----BEGIN X509 CRL-----") + + def _belongs_to(cert_or_pubkey, privkey): if isinstance(cert_or_pubkey, cx509.Certificate): cert_or_pubkey = cert_or_pubkey.public_key() From afaa9e56b64550b51a7a539c2d629673d7787879 Mon Sep 17 00:00:00 2001 From: jeanluc Date: Sat, 6 May 2023 23:58:56 +0200 Subject: [PATCH 007/152] Fix x509_v2 unknown salt-internal kwargs --- changelog/64232.fixed.md | 1 + salt/modules/x509_v2.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 changelog/64232.fixed.md diff --git a/changelog/64232.fixed.md b/changelog/64232.fixed.md new file mode 100644 index 00000000000..45a5ccb90ea --- /dev/null +++ b/changelog/64232.fixed.md @@ -0,0 +1 @@ +Fixed x509_v2 `create_private_key`/`create_crl` unknown kwargs: __pub_fun... diff --git a/salt/modules/x509_v2.py b/salt/modules/x509_v2.py index b46d4cf57d7..0725b1b5624 100644 --- a/salt/modules/x509_v2.py +++ b/salt/modules/x509_v2.py @@ -901,8 +901,11 @@ def create_crl( salt.utils.versions.kwargs_warn_until(["text"], "Potassium") kwargs.pop("text") - if kwargs: - raise SaltInvocationError(f"Unrecognized keyword arguments: {list(kwargs)}") + unknown = [kwarg for kwarg in kwargs if not kwarg.startswith("_")] + if unknown: + raise SaltInvocationError( + f"Unrecognized keyword arguments: {list(unknown)}" + ) if days_valid is None: try: @@ -1235,8 +1238,9 @@ def create_private_key( for x in ignored_params: kwargs.pop(x) - if kwargs: - raise SaltInvocationError(f"Unrecognized keyword arguments: {list(kwargs)}") + unknown = [kwarg for kwarg in kwargs if not kwarg.startswith("_")] + if unknown: + raise SaltInvocationError(f"Unrecognized keyword arguments: {list(unknown)}") if encoding not in ["der", "pem", "pkcs12"]: raise CommandExecutionError( From 16e5b6d24a74d7881e37222b0d65862a81c65500 Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Wed, 26 Apr 2023 14:26:55 -0500 Subject: [PATCH 008/152] Call global logger when catching pip.list exceptions in states.pip.installed --- changelog/64169.fixed.md | 1 + salt/states/pip_state.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog/64169.fixed.md diff --git a/changelog/64169.fixed.md b/changelog/64169.fixed.md new file mode 100644 index 00000000000..499b94b693b --- /dev/null +++ b/changelog/64169.fixed.md @@ -0,0 +1 @@ +Call global logger when catching pip.list exceptions in states.pip.installed diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index 542a7f6c751..fd99d6bd626 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -852,7 +852,7 @@ def installed( ) # If we fail, then just send False, and we'll try again in the next function call except Exception as exc: # pylint: disable=broad-except - log.exception(exc) + globals().get("log").exception(exc) pip_list = False for prefix, state_pkg_name, version_spec in pkgs_details: From 80713947caba26730e541742a016728953a27038 Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Wed, 26 Apr 2023 18:27:11 -0500 Subject: [PATCH 009/152] Add unit test for #64169 --- tests/pytests/unit/states/test_pip.py | 69 +++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/pytests/unit/states/test_pip.py diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py new file mode 100644 index 00000000000..7e04602ce44 --- /dev/null +++ b/tests/pytests/unit/states/test_pip.py @@ -0,0 +1,69 @@ +""" + :codeauthor: Eric Graham +""" +import logging + +import pytest + +import salt.states.pip_state as pip_state +from salt.exceptions import CommandExecutionError +from tests.support.mock import MagicMock, patch + + +@pytest.fixture +def configure_loader_modules(): + return { + pip_state: { + '__env__': 'base', + '__opts__': { + 'test': False + } + } + } + + +def test_issue_64169(caplog): + pkg_to_install = 'nonexistent_package' + exception_message = 'Invalid JSON (test_issue_64169)' + + mock_pip_list = MagicMock(side_effect=[ + CommandExecutionError(exception_message), # pre-cache the pip list (preinstall) + {}, # Checking if the pkg is already installed + {pkg_to_install: '100.10.1'} # Confirming successful installation + ]) + mock_pip_version = MagicMock(return_value='100.10.1') + mock_pip_install = MagicMock(return_value={"retcode": 0, "stdout": ""}) + + with patch.dict(pip_state.__salt__, { + "pip.list": mock_pip_list, + "pip.version": mock_pip_version, + "pip.install": mock_pip_install + }): + with caplog.at_level(logging.WARNING): + # Call pip.installed with a specifically 'broken' pip.list. + # pip.installed should continue, but log the exception from pip.list. + # pip.installed should NOT raise an exception itself. + # noinspection PyBroadException + try: + pip_state.installed( + name=pkg_to_install, + use_wheel=False, # Set False to simplify testing + no_use_wheel=False, # ' + no_binary=False, # ' + log=None # Regression will cause this function call to throw + # an AttributeError + ) + except AttributeError: + # Observed behavior in #64169 + assert False + except: + # Something went wrong, but it isn't what's being tested for here. + return + + # Take 64169 further and actually confirm that the targeted exception from pip.list got logged. + assert exception_message in caplog.messages + + # Confirm that the state continued to install the package as expected. + # Only check the 'pkgs' parameter of pip.install + mock_install_call_args, mock_install_call_kwargs = mock_pip_install.call_args + assert mock_install_call_kwargs['pkgs'] == pkg_to_install From 45369f0367f7b77fe8bf6f45a868d909bf4c7465 Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Mon, 1 May 2023 10:55:29 -0500 Subject: [PATCH 010/152] Rename Global Logger log to logger in pip_state.py --- changelog/64169.fixed.md | 1 + salt/states/pip_state.py | 14 ++++++++------ tests/pytests/unit/states/test_pip.py | 11 +++++++++-- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/changelog/64169.fixed.md b/changelog/64169.fixed.md index 499b94b693b..fe80eff1e94 100644 --- a/changelog/64169.fixed.md +++ b/changelog/64169.fixed.md @@ -1 +1,2 @@ Call global logger when catching pip.list exceptions in states.pip.installed +Rename gloabl logger `log` to `logger` inside pip_state \ No newline at end of file diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index fd99d6bd626..cc5d877c06e 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -114,7 +114,7 @@ if HAS_PIP is True: # pylint: enable=import-error -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) # Define the module's virtual name __virtualname__ = "pip" @@ -189,10 +189,10 @@ def _check_pkg_version_format(pkg): # vcs+URL urls are not properly parsed. # The next line is meant to trigger an AttributeError and # handle lower pip versions - log.debug("Installed pip version: %s", pip.__version__) + logger.debug("Installed pip version: %s", pip.__version__) install_req = _from_line(pkg) except AttributeError: - log.debug("Installed pip version is lower than 1.2") + logger.debug("Installed pip version is lower than 1.2") supported_vcs = ("git", "svn", "hg", "bzr") if pkg.startswith(supported_vcs): for vcs in supported_vcs: @@ -351,7 +351,7 @@ def _pep440_version_cmp(pkg1, pkg2, ignore_epoch=False): making the comparison. """ if HAS_PKG_RESOURCES is False: - log.warning( + logger.warning( "The pkg_resources packages was not loaded. Please install setuptools." ) return None @@ -367,7 +367,7 @@ def _pep440_version_cmp(pkg1, pkg2, ignore_epoch=False): if pkg_resources.parse_version(pkg1) > pkg_resources.parse_version(pkg2): return 1 except Exception as exc: # pylint: disable=broad-except - log.exception(exc) + logger.exception(f'Comparison of package versions "{pkg1}" and "{pkg2}" failed: {exc}') return None @@ -852,7 +852,9 @@ def installed( ) # If we fail, then just send False, and we'll try again in the next function call except Exception as exc: # pylint: disable=broad-except - globals().get("log").exception(exc) + logger.exception( + f'Pre-caching of PIP packages during states.pip.installed failed by exception from pip.list: {exc}' + ) pip_list = False for prefix, state_pkg_name, version_spec in pkgs_details: diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py index 7e04602ce44..1b6d8afb364 100644 --- a/tests/pytests/unit/states/test_pip.py +++ b/tests/pytests/unit/states/test_pip.py @@ -60,8 +60,15 @@ def test_issue_64169(caplog): # Something went wrong, but it isn't what's being tested for here. return - # Take 64169 further and actually confirm that the targeted exception from pip.list got logged. - assert exception_message in caplog.messages + # Take 64169 further and actually confirm that the exception from pip.list got logged. + exc_msg_present = False + for log_line in caplog.messages: + # The exception must be somewhere in the log, but may optionally not be on a line by itself. + if exception_message in log_line: + exc_msg_present = True + break + + assert exc_msg_present # Confirm that the state continued to install the package as expected. # Only check the 'pkgs' parameter of pip.install From 8d682f09b3012a3f41381e2dd321f9418cd44dc6 Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Mon, 1 May 2023 15:51:25 -0500 Subject: [PATCH 011/152] Clarify Failing Test Message; Search for Entire Log Line in caplog --- tests/pytests/unit/states/test_pip.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py index 1b6d8afb364..7d93faa3eb8 100644 --- a/tests/pytests/unit/states/test_pip.py +++ b/tests/pytests/unit/states/test_pip.py @@ -53,22 +53,19 @@ def test_issue_64169(caplog): log=None # Regression will cause this function call to throw # an AttributeError ) - except AttributeError: + except AttributeError as exc: # Observed behavior in #64169 - assert False + pytest.fail( + 'Regression on #64169: pip_state.installed seems to be throwing an unexpected AttributeException: ' + f'{exc}' + ) except: # Something went wrong, but it isn't what's being tested for here. return # Take 64169 further and actually confirm that the exception from pip.list got logged. - exc_msg_present = False - for log_line in caplog.messages: - # The exception must be somewhere in the log, but may optionally not be on a line by itself. - if exception_message in log_line: - exc_msg_present = True - break - - assert exc_msg_present + assert 'Pre-caching of PIP packages during states.pip.installed failed by exception ' \ + f'from pip.list: {exception_message}' in caplog.messages # Confirm that the state continued to install the package as expected. # Only check the 'pkgs' parameter of pip.install From a6d12d59e34ec89d7b50d9e51876e7d5179039f8 Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Mon, 1 May 2023 15:53:52 -0500 Subject: [PATCH 012/152] Fix Changelog Typo --- changelog/64169.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/64169.fixed.md b/changelog/64169.fixed.md index fe80eff1e94..d6ce2bf1937 100644 --- a/changelog/64169.fixed.md +++ b/changelog/64169.fixed.md @@ -1,2 +1,2 @@ Call global logger when catching pip.list exceptions in states.pip.installed -Rename gloabl logger `log` to `logger` inside pip_state \ No newline at end of file +Rename global logger `log` to `logger` inside pip_state \ No newline at end of file From 092cb30f6d02ef54e9712c07dd2838b0d101c0bd Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Mon, 1 May 2023 16:08:55 -0500 Subject: [PATCH 013/152] Remove Silent Catch --- tests/pytests/unit/states/test_pip.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py index 7d93faa3eb8..a7cdc106e62 100644 --- a/tests/pytests/unit/states/test_pip.py +++ b/tests/pytests/unit/states/test_pip.py @@ -59,9 +59,6 @@ def test_issue_64169(caplog): 'Regression on #64169: pip_state.installed seems to be throwing an unexpected AttributeException: ' f'{exc}' ) - except: - # Something went wrong, but it isn't what's being tested for here. - return # Take 64169 further and actually confirm that the exception from pip.list got logged. assert 'Pre-caching of PIP packages during states.pip.installed failed by exception ' \ From 1e4bb8318e141ed98261cf625eb02624f12a9da0 Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Tue, 2 May 2023 13:22:13 -0500 Subject: [PATCH 014/152] Run Black Pre-Commit Step --- salt/states/pip_state.py | 16 ++++--- tests/pytests/unit/states/test_pip.py | 61 ++++++++++++++------------- 2 files changed, 40 insertions(+), 37 deletions(-) diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index cc5d877c06e..de75057adf4 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -251,7 +251,7 @@ def _check_if_installed( index_url, extra_index_url, pip_list=False, - **kwargs + **kwargs, ): """ Takes a package name and version specification (if any) and checks it is @@ -367,7 +367,9 @@ def _pep440_version_cmp(pkg1, pkg2, ignore_epoch=False): if pkg_resources.parse_version(pkg1) > pkg_resources.parse_version(pkg2): return 1 except Exception as exc: # pylint: disable=broad-except - logger.exception(f'Comparison of package versions "{pkg1}" and "{pkg2}" failed: {exc}') + logger.exception( + f'Comparison of package versions "{pkg1}" and "{pkg2}" failed: {exc}' + ) return None @@ -418,7 +420,7 @@ def installed( cache_dir=None, no_binary=None, extra_args=None, - **kwargs + **kwargs, ): """ Make sure the package is installed @@ -853,7 +855,7 @@ def installed( # If we fail, then just send False, and we'll try again in the next function call except Exception as exc: # pylint: disable=broad-except logger.exception( - f'Pre-caching of PIP packages during states.pip.installed failed by exception from pip.list: {exc}' + f"Pre-caching of PIP packages during states.pip.installed failed by exception from pip.list: {exc}" ) pip_list = False @@ -874,7 +876,7 @@ def installed( index_url, extra_index_url, pip_list, - **kwargs + **kwargs, ) # If _check_if_installed result is None, something went wrong with # the command running. This way we keep stateful output. @@ -980,7 +982,7 @@ def installed( no_cache_dir=no_cache_dir, extra_args=extra_args, disable_version_check=True, - **kwargs + **kwargs, ) if pip_install_call and pip_install_call.get("retcode", 1) == 0: @@ -1045,7 +1047,7 @@ def installed( user=user, cwd=cwd, env_vars=env_vars, - **kwargs + **kwargs, ) ) diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py index a7cdc106e62..307ba5e1e65 100644 --- a/tests/pytests/unit/states/test_pip.py +++ b/tests/pytests/unit/states/test_pip.py @@ -12,33 +12,33 @@ from tests.support.mock import MagicMock, patch @pytest.fixture def configure_loader_modules(): - return { - pip_state: { - '__env__': 'base', - '__opts__': { - 'test': False - } - } - } + return {pip_state: {"__env__": "base", "__opts__": {"test": False}}} def test_issue_64169(caplog): - pkg_to_install = 'nonexistent_package' - exception_message = 'Invalid JSON (test_issue_64169)' + pkg_to_install = "nonexistent_package" + exception_message = "Invalid JSON (test_issue_64169)" - mock_pip_list = MagicMock(side_effect=[ - CommandExecutionError(exception_message), # pre-cache the pip list (preinstall) - {}, # Checking if the pkg is already installed - {pkg_to_install: '100.10.1'} # Confirming successful installation - ]) - mock_pip_version = MagicMock(return_value='100.10.1') + mock_pip_list = MagicMock( + side_effect=[ + CommandExecutionError( + exception_message + ), # pre-cache the pip list (preinstall) + {}, # Checking if the pkg is already installed + {pkg_to_install: "100.10.1"}, # Confirming successful installation + ] + ) + mock_pip_version = MagicMock(return_value="100.10.1") mock_pip_install = MagicMock(return_value={"retcode": 0, "stdout": ""}) - with patch.dict(pip_state.__salt__, { - "pip.list": mock_pip_list, - "pip.version": mock_pip_version, - "pip.install": mock_pip_install - }): + with patch.dict( + pip_state.__salt__, + { + "pip.list": mock_pip_list, + "pip.version": mock_pip_version, + "pip.install": mock_pip_install, + }, + ): with caplog.at_level(logging.WARNING): # Call pip.installed with a specifically 'broken' pip.list. # pip.installed should continue, but log the exception from pip.list. @@ -47,24 +47,25 @@ def test_issue_64169(caplog): try: pip_state.installed( name=pkg_to_install, - use_wheel=False, # Set False to simplify testing + use_wheel=False, # Set False to simplify testing no_use_wheel=False, # ' - no_binary=False, # ' - log=None # Regression will cause this function call to throw - # an AttributeError + no_binary=False, # ' + log=None, # Regression will cause this function call to throw an AttributeError ) except AttributeError as exc: # Observed behavior in #64169 pytest.fail( - 'Regression on #64169: pip_state.installed seems to be throwing an unexpected AttributeException: ' - f'{exc}' + "Regression on #64169: pip_state.installed seems to be throwing an unexpected AttributeException: " + f"{exc}" ) # Take 64169 further and actually confirm that the exception from pip.list got logged. - assert 'Pre-caching of PIP packages during states.pip.installed failed by exception ' \ - f'from pip.list: {exception_message}' in caplog.messages + assert ( + "Pre-caching of PIP packages during states.pip.installed failed by exception " + f"from pip.list: {exception_message}" in caplog.messages + ) # Confirm that the state continued to install the package as expected. # Only check the 'pkgs' parameter of pip.install mock_install_call_args, mock_install_call_kwargs = mock_pip_install.call_args - assert mock_install_call_kwargs['pkgs'] == pkg_to_install + assert mock_install_call_kwargs["pkgs"] == pkg_to_install From 874230620870aaf3eb99e38e78a9451291418ffe Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Wed, 3 May 2023 09:39:26 -0500 Subject: [PATCH 015/152] Add New Line to Changelog --- changelog/64169.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/64169.fixed.md b/changelog/64169.fixed.md index d6ce2bf1937..e8631285aaa 100644 --- a/changelog/64169.fixed.md +++ b/changelog/64169.fixed.md @@ -1,2 +1,2 @@ Call global logger when catching pip.list exceptions in states.pip.installed -Rename global logger `log` to `logger` inside pip_state \ No newline at end of file +Rename global logger `log` to `logger` inside pip_state From 4f14710652c59d7fbae072d9dfc3094e8f9e2fe4 Mon Sep 17 00:00:00 2001 From: ScriptAutomate Date: Wed, 10 May 2023 14:02:43 -0500 Subject: [PATCH 016/152] Update banners and links --- doc/_themes/saltstack2/layout.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/_themes/saltstack2/layout.html b/doc/_themes/saltstack2/layout.html index 04bff89e1fb..001844f7cd2 100644 --- a/doc/_themes/saltstack2/layout.html +++ b/doc/_themes/saltstack2/layout.html @@ -152,7 +152,7 @@ - +
- +
{% endif %} @@ -295,7 +295,7 @@ {% else %} {% endif %} #}--> - + {% if build_type=="next" %} From b897734f4a33af89b814c39eb62cd48388776705 Mon Sep 17 00:00:00 2001 From: Alex Dehnert Date: Fri, 14 Apr 2023 23:16:02 -0400 Subject: [PATCH 017/152] Add warning about effective rights mask Group permission on the file should generally be at least as broad as any file ACLs, to avoid ineffective ACLs and/or changes each time the state is run. --- salt/states/linux_acl.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/salt/states/linux_acl.py b/salt/states/linux_acl.py index c45c9383487..55157457486 100644 --- a/salt/states/linux_acl.py +++ b/salt/states/linux_acl.py @@ -50,6 +50,24 @@ Ensure a Linux ACL list does not exist - damian - homer - perms: rwx + +.. warning:: + + The effective permissions of Linux file access control lists (ACLs) are + governed by the "effective rights mask" (the `mask` line in the output of + the `getfacl` command) combined with the `perms` set by this module: any + permission bits (for example, r=read) present in an ACL but not in the mask + are ignored. The mask is automatically recomputed when setting an ACL, so + normally this isn't important. However, if the file permissions are + changed (with `chmod` or `file.managed`, for example), the mask will + generally be set based on just the group bits of the file permissions. + + As a result, when using `file.managed` or similar to control file + permissions as well as this module, you should set your group permissions + to be at least as broad as any permissions in your ACL. Otherwise, the two + state declarations will each register changes each run, and if the `file` + declaration runs later, your ACL will be ineffective. + """ From a5cfe80c89ee6996dfef55630f20b4100c693dff Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 19 May 2023 02:04:24 -0700 Subject: [PATCH 018/152] Ubuntu pkg tests run apt non-interactive mode. Issue #64307 --- pkg/tests/support/helpers.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/tests/support/helpers.py b/pkg/tests/support/helpers.py index 57b6ccd4d00..f4f26f0781a 100644 --- a/pkg/tests/support/helpers.py +++ b/pkg/tests/support/helpers.py @@ -596,8 +596,26 @@ class SaltPkgInstall: self.proc.run("launchctl", "disable", f"system/{service_name}") self.proc.run("launchctl", "bootout", "system", str(plist_file)) elif upgrade: + env = os.environ.copy() + extra_args = [] + if self.distro_id in ("ubuntu", "debian"): + env["DEBIAN_FRONTEND"] = "noninteractive" + extra_args = [ + "-o", + "DPkg::Options::=--force-confdef", + "-o", + "DPkg::Options::=--force-confold", + ] log.info("Installing packages:\n%s", pprint.pformat(self.pkgs)) - ret = self.proc.run(self.pkg_mngr, "upgrade", "-y", *self.pkgs) + args = extra_args + self.pkgs + ret = self.proc.run( + self.pkg_mngr, + "upgrade", + "-y", + *args, + _timeout=120, + env=env, + ) else: log.info("Installing packages:\n%s", pprint.pformat(self.pkgs)) ret = self.proc.run(self.pkg_mngr, "install", "-y", *self.pkgs) From 50c666eaf8cad76c3b384fbe3df439ef8442d810 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 22 May 2023 03:15:56 -0700 Subject: [PATCH 019/152] Check return code instead of stdout --- pkg/tests/support/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tests/support/helpers.py b/pkg/tests/support/helpers.py index f4f26f0781a..9853c441870 100644 --- a/pkg/tests/support/helpers.py +++ b/pkg/tests/support/helpers.py @@ -621,7 +621,7 @@ class SaltPkgInstall: ret = self.proc.run(self.pkg_mngr, "install", "-y", *self.pkgs) if not platform.is_darwin() and not platform.is_windows(): # Make sure we don't have any trailing references to old package file locations - assert "No such file or directory" not in ret.stdout + ret.returncode == 0 assert "/saltstack/salt/run" not in ret.stdout log.info(ret) self._check_retcode(ret) From f4af592400fc1d6b1006477025a0322248b815c1 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 22 May 2023 16:35:31 +0100 Subject: [PATCH 020/152] Sometimes the first page does not have any results. Try next page if there's a next token. Signed-off-by: Pedro Algarvio --- tools/vm.py | 75 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/tools/vm.py b/tools/vm.py index 944f2fe6cc2..f7b2837ae1b 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -720,41 +720,50 @@ class VM: client = boto3.client("ec2", region_name=self.region_name) # Let's search for the launch template corresponding to this AMI launch_template_name = None + next_token = "" try: - response = response = client.describe_launch_templates( - Filters=[ - { - "Name": "tag:spb:is-golden-image-template", - "Values": ["true"], - }, - { - "Name": "tag:spb:project", - "Values": ["salt-project"], - }, - { - "Name": "tag:spb:environment", - "Values": [environment], - }, - { - "Name": "tag:spb:image-id", - "Values": [self.config.ami], - }, - ] - ) - log.debug( - "Search for launch template response:\n%s", pprint.pformat(response) - ) - for details in response.get("LaunchTemplates"): - if launch_template_name is not None: - log.warning( - "Multiple launch templates for the same AMI. This is not " - "supposed to happen. Picked the first one listed: %s", - response, - ) - break - launch_template_name = details["LaunchTemplateName"] + while True: + response = response = client.describe_launch_templates( + Filters=[ + { + "Name": "tag:spb:is-golden-image-template", + "Values": ["true"], + }, + { + "Name": "tag:spb:project", + "Values": ["salt-project"], + }, + { + "Name": "tag:spb:environment", + "Values": [environment], + }, + { + "Name": "tag:spb:image-id", + "Values": [self.config.ami], + }, + ], + NextToken=next_token, + ) + log.debug( + "Search for launch template response:\n%s", + pprint.pformat(response), + ) + for details in response.get("LaunchTemplates"): + if launch_template_name is not None: + log.warning( + "Multiple launch templates for the same AMI. This is not " + "supposed to happen. Picked the first one listed: %s", + response, + ) + break + launch_template_name = details["LaunchTemplateName"] - if launch_template_name is None: + if launch_template_name is not None: + break + + next_token = response.get("NextToken") + if next_token: + continue self.ctx.error(f"Could not find a launch template for {self.name!r}") self.ctx.exit(1) except ClientError as exc: From 15849a5911e4e9a7e34b9f2998330b5b1b0db911 Mon Sep 17 00:00:00 2001 From: natalieswork Date: Mon, 22 May 2023 14:21:25 -0400 Subject: [PATCH 021/152] removing azure code from repo --- .../clouds/all/salt.cloud.clouds.azurearm.rst | 5 - .../clouds/all/salt.cloud.clouds.msazure.rst | 5 - .../all/salt.fileserver.azurefs.rst | 4 - .../grains/all/salt.grains.metadata_azure.rst | 5 - .../all/salt.modules.azurearm_compute.rst | 5 - .../modules/all/salt.modules.azurearm_dns.rst | 6 - .../all/salt.modules.azurearm_network.rst | 5 - .../all/salt.modules.azurearm_resource.rst | 5 - doc/ref/pillar/all/salt.pillar.azureblob.rst | 5 - .../all/salt.states.azurearm_compute.rst | 5 - .../states/all/salt.states.azurearm_dns.rst | 5 - .../all/salt.states.azurearm_network.rst | 5 - .../all/salt.states.azurearm_resource.rst | 5 - doc/topics/cloud/azure.rst | 1481 ------- doc/topics/cloud/azurearm.rst | 486 --- salt/cloud/clouds/azurearm.py | 2043 --------- salt/cloud/clouds/msazure.py | 3665 ----------------- salt/fileserver/azurefs.py | 396 -- salt/grains/metadata_azure.py | 45 - salt/modules/azurearm_compute.py | 754 ---- salt/modules/azurearm_dns.py | 552 --- salt/modules/azurearm_network.py | 2859 ------------- salt/modules/azurearm_resource.py | 1253 ------ salt/pillar/azureblob.py | 465 --- salt/states/azurearm_compute.py | 362 -- salt/states/azurearm_dns.py | 762 ---- salt/states/azurearm_network.py | 2594 ------------ salt/states/azurearm_resource.py | 880 ---- salt/utils/azurearm.py | 338 -- salt/utils/msazure.py | 189 - .../integration/cloud/clouds/test_msazure.py | 66 - .../files/conf/cloud.profiles.d/azure.conf | 8 - .../conf/cloud.providers.d/azurearm.conf | 16 - .../unit/cloud/clouds/test_azurearm.py | 161 - .../unit/grains/test_metadata_azure.py | 96 - .../pytests/unit/modules/test_azurearm_dns.py | 182 - tests/pytests/unit/pillar/test_azureblob.py | 333 -- tests/unit/utils/test_azurearm.py | 55 - 38 files changed, 20106 deletions(-) delete mode 100644 doc/ref/clouds/all/salt.cloud.clouds.azurearm.rst delete mode 100644 doc/ref/clouds/all/salt.cloud.clouds.msazure.rst delete mode 100644 doc/ref/file_server/all/salt.fileserver.azurefs.rst delete mode 100644 doc/ref/grains/all/salt.grains.metadata_azure.rst delete mode 100644 doc/ref/modules/all/salt.modules.azurearm_compute.rst delete mode 100644 doc/ref/modules/all/salt.modules.azurearm_dns.rst delete mode 100644 doc/ref/modules/all/salt.modules.azurearm_network.rst delete mode 100644 doc/ref/modules/all/salt.modules.azurearm_resource.rst delete mode 100644 doc/ref/pillar/all/salt.pillar.azureblob.rst delete mode 100644 doc/ref/states/all/salt.states.azurearm_compute.rst delete mode 100644 doc/ref/states/all/salt.states.azurearm_dns.rst delete mode 100644 doc/ref/states/all/salt.states.azurearm_network.rst delete mode 100644 doc/ref/states/all/salt.states.azurearm_resource.rst delete mode 100644 doc/topics/cloud/azure.rst delete mode 100644 doc/topics/cloud/azurearm.rst delete mode 100644 salt/cloud/clouds/azurearm.py delete mode 100644 salt/cloud/clouds/msazure.py delete mode 100644 salt/fileserver/azurefs.py delete mode 100644 salt/grains/metadata_azure.py delete mode 100644 salt/modules/azurearm_compute.py delete mode 100644 salt/modules/azurearm_dns.py delete mode 100644 salt/modules/azurearm_network.py delete mode 100644 salt/modules/azurearm_resource.py delete mode 100644 salt/pillar/azureblob.py delete mode 100644 salt/states/azurearm_compute.py delete mode 100644 salt/states/azurearm_dns.py delete mode 100644 salt/states/azurearm_network.py delete mode 100644 salt/states/azurearm_resource.py delete mode 100644 salt/utils/azurearm.py delete mode 100644 salt/utils/msazure.py delete mode 100644 tests/integration/cloud/clouds/test_msazure.py delete mode 100644 tests/integration/files/conf/cloud.profiles.d/azure.conf delete mode 100644 tests/integration/files/conf/cloud.providers.d/azurearm.conf delete mode 100644 tests/pytests/unit/cloud/clouds/test_azurearm.py delete mode 100644 tests/pytests/unit/grains/test_metadata_azure.py delete mode 100644 tests/pytests/unit/modules/test_azurearm_dns.py delete mode 100644 tests/pytests/unit/pillar/test_azureblob.py delete mode 100644 tests/unit/utils/test_azurearm.py diff --git a/doc/ref/clouds/all/salt.cloud.clouds.azurearm.rst b/doc/ref/clouds/all/salt.cloud.clouds.azurearm.rst deleted file mode 100644 index 86c4b7eea76..00000000000 --- a/doc/ref/clouds/all/salt.cloud.clouds.azurearm.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.cloud.clouds.azurearm -========================== - -.. automodule:: salt.cloud.clouds.azurearm - :members: diff --git a/doc/ref/clouds/all/salt.cloud.clouds.msazure.rst b/doc/ref/clouds/all/salt.cloud.clouds.msazure.rst deleted file mode 100644 index 4b624291f46..00000000000 --- a/doc/ref/clouds/all/salt.cloud.clouds.msazure.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.cloud.clouds.msazure -========================= - -.. automodule:: salt.cloud.clouds.msazure - :members: diff --git a/doc/ref/file_server/all/salt.fileserver.azurefs.rst b/doc/ref/file_server/all/salt.fileserver.azurefs.rst deleted file mode 100644 index c1291ed9890..00000000000 --- a/doc/ref/file_server/all/salt.fileserver.azurefs.rst +++ /dev/null @@ -1,4 +0,0 @@ -salt.fileserver.azurefs -======================= - -.. automodule:: salt.fileserver.azurefs diff --git a/doc/ref/grains/all/salt.grains.metadata_azure.rst b/doc/ref/grains/all/salt.grains.metadata_azure.rst deleted file mode 100644 index 41cfa502969..00000000000 --- a/doc/ref/grains/all/salt.grains.metadata_azure.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.grains.metadata_azure -========================== - -.. automodule:: salt.grains.metadata_azure - :members: diff --git a/doc/ref/modules/all/salt.modules.azurearm_compute.rst b/doc/ref/modules/all/salt.modules.azurearm_compute.rst deleted file mode 100644 index f82507d2f99..00000000000 --- a/doc/ref/modules/all/salt.modules.azurearm_compute.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.modules.azurearm_compute -============================= - -.. automodule:: salt.modules.azurearm_compute - :members: diff --git a/doc/ref/modules/all/salt.modules.azurearm_dns.rst b/doc/ref/modules/all/salt.modules.azurearm_dns.rst deleted file mode 100644 index 23e33fa0ab9..00000000000 --- a/doc/ref/modules/all/salt.modules.azurearm_dns.rst +++ /dev/null @@ -1,6 +0,0 @@ -salt.modules.azurearm_dns -========================= - -.. automodule:: salt.modules.azurearm_dns - :members: - :undoc-members: diff --git a/doc/ref/modules/all/salt.modules.azurearm_network.rst b/doc/ref/modules/all/salt.modules.azurearm_network.rst deleted file mode 100644 index 957649d3ea7..00000000000 --- a/doc/ref/modules/all/salt.modules.azurearm_network.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.modules.azurearm_network -============================= - -.. automodule:: salt.modules.azurearm_network - :members: diff --git a/doc/ref/modules/all/salt.modules.azurearm_resource.rst b/doc/ref/modules/all/salt.modules.azurearm_resource.rst deleted file mode 100644 index c020a588350..00000000000 --- a/doc/ref/modules/all/salt.modules.azurearm_resource.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.modules.azurearm_resource -============================== - -.. automodule:: salt.modules.azurearm_resource - :members: diff --git a/doc/ref/pillar/all/salt.pillar.azureblob.rst b/doc/ref/pillar/all/salt.pillar.azureblob.rst deleted file mode 100644 index f47f5929d5f..00000000000 --- a/doc/ref/pillar/all/salt.pillar.azureblob.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.pillar.azureblob -===================== - -.. automodule:: salt.pillar.azureblob - :members: diff --git a/doc/ref/states/all/salt.states.azurearm_compute.rst b/doc/ref/states/all/salt.states.azurearm_compute.rst deleted file mode 100644 index 0f545fed4ef..00000000000 --- a/doc/ref/states/all/salt.states.azurearm_compute.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.states.azurearm_compute -============================ - -.. automodule:: salt.states.azurearm_compute - :members: diff --git a/doc/ref/states/all/salt.states.azurearm_dns.rst b/doc/ref/states/all/salt.states.azurearm_dns.rst deleted file mode 100644 index 29ebdb16d3d..00000000000 --- a/doc/ref/states/all/salt.states.azurearm_dns.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.states.azurearm_dns -======================== - -.. automodule:: salt.states.azurearm_dns - :members: diff --git a/doc/ref/states/all/salt.states.azurearm_network.rst b/doc/ref/states/all/salt.states.azurearm_network.rst deleted file mode 100644 index adc0754716e..00000000000 --- a/doc/ref/states/all/salt.states.azurearm_network.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.states.azurearm_network -============================ - -.. automodule:: salt.states.azurearm_network - :members: diff --git a/doc/ref/states/all/salt.states.azurearm_resource.rst b/doc/ref/states/all/salt.states.azurearm_resource.rst deleted file mode 100644 index 4ea24a8dde3..00000000000 --- a/doc/ref/states/all/salt.states.azurearm_resource.rst +++ /dev/null @@ -1,5 +0,0 @@ -salt.states.azurearm_resource -============================= - -.. automodule:: salt.states.azurearm_resource - :members: diff --git a/doc/topics/cloud/azure.rst b/doc/topics/cloud/azure.rst deleted file mode 100644 index 24598dec2e6..00000000000 --- a/doc/topics/cloud/azure.rst +++ /dev/null @@ -1,1481 +0,0 @@ -========================== -Getting Started With Azure -========================== - -.. versionadded:: 2014.1.0 - -.. warning:: - - This cloud provider will be removed from Salt in version 3007 due to - the deprecation of the "Classic" API for Azure. Please migrate to - `Azure Resource Manager by March 1, 2023 - `_ - -Azure is a cloud service by Microsoft providing virtual machines, SQL services, -media services, and more. This document describes how to use Salt Cloud to -create a virtual machine on Azure, with Salt installed. - -More information about Azure is located at `http://www.windowsazure.com/ -`_. - - -Dependencies -============ -* `Microsoft Azure SDK for Python `_ >= 1.0.2 -* The python-requests library, for Python < 2.7.9. -* A Microsoft Azure account -* OpenSSL (to generate the certificates) -* `Salt `_ - - -Configuration -============= - -Set up the provider config at ``/etc/salt/cloud.providers.d/azure.conf``: - -.. code-block:: yaml - - # Note: This example is for /etc/salt/cloud.providers.d/azure.conf - - my-azure-config: - driver: azure - subscription_id: 3287abc8-f98a-c678-3bde-326766fd3617 - certificate_path: /etc/salt/azure.pem - - # Set up the location of the salt master - # - minion: - master: saltmaster.example.com - - # Optional - management_host: management.core.windows.net - -The certificate used must be generated by the user. OpenSSL can be used to -create the management certificates. Two certificates are needed: a .cer file, -which is uploaded to Azure, and a .pem file, which is stored locally. - -To create the .pem file, execute the following command: - -.. code-block:: bash - - openssl req -x509 -nodes -days 365 -newkey rsa:1024 -keyout /etc/salt/azure.pem -out /etc/salt/azure.pem - -To create the .cer file, execute the following command: - -.. code-block:: bash - - openssl x509 -inform pem -in /etc/salt/azure.pem -outform der -out /etc/salt/azure.cer - -After creating these files, the .cer file will need to be uploaded to -Azure via the "Upload a Management Certificate" action of the "Management Certificates" -tab within the "Settings" section of the management portal. - -Optionally, a ``management_host`` may be configured, if necessary for the region. - -.. note:: - .. versionchanged:: 2015.8.0 - - The ``provider`` parameter in cloud provider definitions was renamed to ``driver``. This - change was made to avoid confusion with the ``provider`` parameter that is used in cloud profile - definitions. Cloud provider definitions now use ``driver`` to refer to the Salt cloud module that - provides the underlying functionality to connect to a cloud host, while cloud profiles continue - to use ``provider`` to refer to provider configurations that you define. - -Cloud Profiles -============== -Set up an initial profile at ``/etc/salt/cloud.profiles``: - -.. code-block:: yaml - - azure-ubuntu: - provider: my-azure-config - image: 'b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-12_04_3-LTS-amd64-server-20131003-en-us-30GB' - size: Small - location: 'East US' - ssh_username: azureuser - ssh_password: verybadpass - slot: production - media_link: 'http://portalvhdabcdefghijklmn.blob.core.windows.net/vhds' - virtual_network_name: azure-virtual-network - subnet_name: azure-subnet - -These options are described in more detail below. Once configured, the profile -can be realized with a salt command: - -.. code-block:: bash - - salt-cloud -p azure-ubuntu newinstance - -This will create an salt minion instance named ``newinstance`` in Azure. If -the command was executed on the salt-master, its Salt key will automatically -be signed on the master. - -Once the instance has been created with salt-minion installed, connectivity to -it can be verified with Salt: - -.. code-block:: bash - - salt newinstance test.version - - -Profile Options -=============== -The following options are currently available for Azure. - -provider --------- -The name of the provider as configured in -`/etc/salt/cloud.providers.d/azure.conf`. - -image ------ -The name of the image to use to create a VM. Available images can be viewed -using the following command: - -.. code-block:: bash - - salt-cloud --list-images my-azure-config - -size ----- -The name of the size to use to create a VM. Available sizes can be viewed using -the following command: - -.. code-block:: bash - - salt-cloud --list-sizes my-azure-config - -location --------- -The name of the location to create a VM in. Available locations can be viewed -using the following command: - -.. code-block:: bash - - salt-cloud --list-locations my-azure-config - -affinity_group --------------- -The name of the affinity group to create a VM in. Either a ``location`` or an -``affinity_group`` may be specified, but not both. See Affinity Groups below. - -ssh_username ------------- -The user to use to log into the newly-created VM to install Salt. - -ssh_password ------------- -The password to use to log into the newly-created VM to install Salt. - -slot ----- -The environment to which the hosted service is deployed. Valid values are -`staging` or `production`. When set to `production`, the resulting URL of the -new VM will be `.cloudapp.net`. When set to `staging`, the resulting -URL will contain a generated hash instead. - -media_link ----------- -This is the URL of the container that will store the disk that this VM uses. -Currently, this container must already exist. If a VM has previously been -created in the associated account, a container should already exist. In the web -interface, go into the Storage area and click one of the available storage -selections. Click the Containers link, and then copy the URL from the container -that will be used. It generally looks like: - -.. code-block:: yaml - - http://portalvhdabcdefghijklmn.blob.core.windows.net/vhds - -service_name ------------- -The name of the service in which to create the VM. If this is not specified, -then a service will be created with the same name as the VM. - -virtual_network_name --------------------- -Optional. The name of the virtual network for the VM to join. If this is not -specified, then no virtual network will be joined. - -subnet_name ------------- -Optional. The name of the subnet in the virtual network for the VM to join. -Requires that a ``virtual_network_name`` is specified. - - -Show Instance -============= -This action is a thin wrapper around ``--full-query``, which displays details on -a single instance only. In an environment with several machines, this will save -a user from having to sort through all instance data, just to examine a single -instance. - -.. code-block:: bash - - salt-cloud -a show_instance myinstance - - -Destroying VMs -============== -There are certain options which can be specified in the global cloud -configuration file (usually ``/etc/salt/cloud``) which affect Salt Cloud's -behavior when a VM is destroyed. - -cleanup_disks -------------- -.. versionadded:: 2015.8.0 - -Default is ``False``. When set to ``True``, Salt Cloud will wait for the VM to -be destroyed, then attempt to destroy the main disk that is associated with the -VM. - -cleanup_vhds ------------- -.. versionadded:: 2015.8.0 - -Default is ``False``. Requires ``cleanup_disks`` to be set to ``True``. When -also set to ``True``, Salt Cloud will ask Azure to delete the VHD associated -with the disk that is also destroyed. - -cleanup_services ----------------- -.. versionadded:: 2015.8.0 - -Default is ``False``. Requires ``cleanup_disks`` to be set to ``True``. When -also set to ``True``, Salt Cloud will wait for the disk to be destroyed, then -attempt to remove the service that is associated with the VM. Because the disk -belongs to the service, the disk must be destroyed before the service can be. - - -Managing Hosted Services -======================== -.. versionadded:: 2015.8.0 - -An account can have one or more hosted services. A hosted service is required -in order to create a VM. However, as mentioned above, if a hosted service is not -specified when a VM is created, then one will automatically be created with the -name of the name. The following functions are also available. - -create_service --------------- -Create a hosted service. The following options are available. - -name -~~~~ -Required. The name of the hosted service to create. - -label -~~~~~ -Required. A label to apply to the hosted service. - -description -~~~~~~~~~~~ -Optional. A longer description of the hosted service. - -location -~~~~~~~~ -Required, if ``affinity_group`` is not set. The location in which to create the -hosted service. Either the ``location`` or the ``affinity_group`` must be set, -but not both. - -affinity_group -~~~~~~~~~~~~~~ -Required, if ``location`` is not set. The affinity group in which to create the -hosted service. Either the ``location`` or the ``affinity_group`` must be set, -but not both. - -extended_properties -~~~~~~~~~~~~~~~~~~~ -Optional. Dictionary containing name/value pairs of hosted service properties. -You can have a maximum of 50 extended property name/value pairs. The maximum -length of the Name element is 64 characters, only alphanumeric characters and -underscores are valid in the Name, and the name must start with a letter. -The value has a maximum length of 255 characters. - -CLI Example -~~~~~~~~~~~ -The following example illustrates creating a hosted service. - -.. code-block:: bash - - salt-cloud -f create_service my-azure name=my-service label=my-service location='West US' - -show_service ------------- -Return details about a specific hosted service. Can also be called with -``get_service``. - -.. code-block:: bash - - salt-cloud -f show_storage my-azure name=my-service - -list_services -------------- -List all hosted services associates with the subscription. - -.. code-block:: bash - - salt-cloud -f list_services my-azure-config - - -delete_service --------------- -Delete a specific hosted service. - -.. code-block:: bash - - salt-cloud -f delete_service my-azure name=my-service - - -Managing Storage Accounts -========================= -.. versionadded:: 2015.8.0 - -Salt Cloud can manage storage accounts associated with the account. The -following functions are available. Deprecated marked as deprecated are marked -as such as per the SDK documentation, but are still included for completeness -with the SDK. - -create_storage --------------- -Create a storage account. The following options are supported. - -name -~~~~ -Required. The name of the storage account to create. - -label -~~~~~ -Required. A label to apply to the storage account. - -description -~~~~~~~~~~~ -Optional. A longer description of the storage account. - -location -~~~~~~~~ -Required, if ``affinity_group`` is not set. The location in which to create the -storage account. Either the ``location`` or the ``affinity_group`` must be set, -but not both. - -affinity_group -~~~~~~~~~~~~~~ -Required, if ``location`` is not set. The affinity group in which to create the -storage account. Either the ``location`` or the ``affinity_group`` must be set, -but not both. - -extended_properties -~~~~~~~~~~~~~~~~~~~ -Optional. Dictionary containing name/value pairs of storage account properties. -You can have a maximum of 50 extended property name/value pairs. The maximum -length of the Name element is 64 characters, only alphanumeric characters and -underscores are valid in the Name, and the name must start with a letter. The -value has a maximum length of 255 characters. - -geo_replication_enabled -~~~~~~~~~~~~~~~~~~~~~~~ -Deprecated. Replaced by the account_type parameter. - -account_type -~~~~~~~~~~~~ -Specifies whether the account supports locally-redundant storage, geo-redundant -storage, zone-redundant storage, or read access geo-redundant storage. Possible -values are: - -- Standard_LRS -- Standard_ZRS -- Standard_GRS -- Standard_RAGRS - -CLI Example -~~~~~~~~~~~ -The following example illustrates creating a storage account. - -.. code-block:: bash - - salt-cloud -f create_storage my-azure name=my-storage label=my-storage location='West US' - -list_storage ------------- -List all storage accounts associates with the subscription. - -.. code-block:: bash - - salt-cloud -f list_storage my-azure-config - -show_storage ------------- -Return details about a specific storage account. Can also be called with -``get_storage``. - -.. code-block:: bash - - salt-cloud -f show_storage my-azure name=my-storage - -update_storage --------------- -Update details concerning a storage account. Any of the options available in -``create_storage`` can be used, but the name cannot be changed. - -.. code-block:: bash - - salt-cloud -f update_storage my-azure name=my-storage label=my-storage - -delete_storage --------------- -Delete a specific storage account. - -.. code-block:: bash - - salt-cloud -f delete_storage my-azure name=my-storage - -show_storage_keys ------------------ -Returns the primary and secondary access keys for the specified storage account. - -.. code-block:: bash - - salt-cloud -f show_storage_keys my-azure name=my-storage - -regenerate_storage_keys ------------------------ -Regenerate storage account keys. Requires a key_type ("primary" or "secondary") -to be specified. - -.. code-block:: bash - - salt-cloud -f regenerate_storage_keys my-azure name=my-storage key_type=primary - - -Managing Disks -============== -.. versionadded:: 2015.8.0 - -When a VM is created, a disk will also be created for it. The following -functions are available for managing disks. Deprecated marked as deprecated are -marked as such as per the SDK documentation, but are still included for -completeness with the SDK. - -show_disk ---------- -Return details about a specific disk. Can also be called with ``get_disk``. - -.. code-block:: bash - - salt-cloud -f show_disk my-azure name=my-disk - -list_disks ----------- -List all disks associates with the account. - -.. code-block:: bash - - salt-cloud -f list_disks my-azure - -update_disk ------------ -Update details for a disk. The following options are available. - -name -~~~~ -Required. The name of the disk to update. - -has_operating_system -~~~~~~~~~~~~~~~~~~~~ -Deprecated. - -label -~~~~~ -Required. The label for the disk. - -media_link -~~~~~~~~~~ -Deprecated. The location of the disk in the account, including the storage -container that it is in. This should not need to be changed. - -new_name -~~~~~~~~ -Deprecated. If renaming the disk, the new name. - -os -~~~ -Deprecated. - -CLI Example -~~~~~~~~~~~ -The following example illustrates updating a disk. - -.. code-block:: bash - - salt-cloud -f update_disk my-azure name=my-disk label=my-disk - -delete_disk ------------ -Delete a specific disk. - -.. code-block:: bash - - salt-cloud -f delete_disk my-azure name=my-disk - - -Managing Service Certificates -============================= -.. versionadded:: 2015.8.0 - -Stored at the cloud service level, these certificates are used by your deployed -services. For more information on service certificates, see the following link: - -* `Manage Certificates`__ - -.. __: https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-certs-create - -The following functions are available. - -list_service_certificates -------------------------- -List service certificates associated with the account. - -.. code-block:: bash - - salt-cloud -f list_service_certificates my-azure - -show_service_certificate ------------------------- -Show the data for a specific service certificate associated with the account. -The ``name``, ``thumbprint``, and ``thumbalgorithm`` can be obtained from -``list_service_certificates``. Can also be called with -``get_service_certificate``. - -.. code-block:: bash - - salt-cloud -f show_service_certificate my-azure name=my_service_certificate \ - thumbalgorithm=sha1 thumbprint=0123456789ABCDEF - -add_service_certificate ------------------------ -Add a service certificate to the account. This requires that a certificate -already exists, which is then added to the account. For more information on -creating the certificate itself, see: - -* `Create a Service Certificate for Azure`__ - -.. __: https://msdn.microsoft.com/en-us/library/azure/gg432987.aspx - -The following options are available. - -name -~~~~ -Required. The name of the hosted service that the certificate will belong to. - -data -~~~~ -Required. The base-64 encoded form of the pfx file. - -certificate_format -~~~~~~~~~~~~~~~~~~ -Required. The service certificate format. The only supported value is pfx. - -password -~~~~~~~~ -The certificate password. - -.. code-block:: bash - - salt-cloud -f add_service_certificate my-azure name=my-cert \ - data='...CERT_DATA...' certificate_format=pfx password=verybadpass - -delete_service_certificate --------------------------- -Delete a service certificate from the account. The ``name``, ``thumbprint``, -and ``thumbalgorithm`` can be obtained from ``list_service_certificates``. - -.. code-block:: bash - - salt-cloud -f delete_service_certificate my-azure \ - name=my_service_certificate \ - thumbalgorithm=sha1 thumbprint=0123456789ABCDEF - - -Managing Management Certificates -================================ -.. versionadded:: 2015.8.0 - -A Azure management certificate is an X.509 v3 certificate used to authenticate -an agent, such as Visual Studio Tools for Windows Azure or a client application -that uses the Service Management API, acting on behalf of the subscription owner -to manage subscription resources. Azure management certificates are uploaded to -Azure and stored at the subscription level. The management certificate store can -hold up to 100 certificates per subscription. These certificates are used to -authenticate your Windows Azure deployment. - -For more information on management certificates, see the following link. - -* `Manage Certificates`__ - -.. __: https://msdn.microsoft.com/en-us/library/azure/gg981929.aspx - -The following functions are available. - -list_management_certificates ----------------------------- -List management certificates associated with the account. - -.. code-block:: bash - - salt-cloud -f list_management_certificates my-azure - -show_management_certificate ---------------------------- -Show the data for a specific management certificate associated with the account. -The ``name``, ``thumbprint``, and ``thumbalgorithm`` can be obtained from -``list_management_certificates``. Can also be called with -``get_management_certificate``. - -.. code-block:: bash - - salt-cloud -f show_management_certificate my-azure name=my_management_certificate \ - thumbalgorithm=sha1 thumbprint=0123456789ABCDEF - -add_management_certificate --------------------------- -Management certificates must have a key length of at least 2048 bits and should -reside in the Personal certificate store. When the certificate is installed on -the client, it should contain the private key of the certificate. To upload to -the certificate to the Microsoft Azure Management Portal, you must export it as -a .cer format file that does not contain the private key. For more information -on creating management certificates, see the following link: - -* `Create and Upload a Management Certificate for Azure`__ - -.. __: https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-certs-create - -The following options are available. - -public_key -~~~~~~~~~~ -A base64 representation of the management certificate public key. - -thumbprint -~~~~~~~~~~ -The thumb print that uniquely identifies the management certificate. - -data -~~~~ -The certificate's raw data in base-64 encoded .cer format. - -.. code-block:: bash - - salt-cloud -f add_management_certificate my-azure public_key='...PUBKEY...' \ - thumbprint=0123456789ABCDEF data='...CERT_DATA...' - -delete_management_certificate ------------------------------ -Delete a management certificate from the account. The ``thumbprint`` can be -obtained from ``list_management_certificates``. - -.. code-block:: bash - - salt-cloud -f delete_management_certificate my-azure thumbprint=0123456789ABCDEF - - -Virtual Network Management -========================== -.. versionadded:: 2015.8.0 - -The following are functions for managing virtual networks. - -list_virtual_networks ---------------------- -List input endpoints associated with the deployment. - -.. code-block:: bash - - salt-cloud -f list_virtual_networks my-azure service=myservice deployment=mydeployment - - -Managing Input Endpoints -======================== -.. versionadded:: 2015.8.0 - -Input endpoints are used to manage port access for roles. Because endpoints -cannot be managed by the Azure Python SDK, Salt Cloud uses the API directly. -With versions of Python before 2.7.9, the ``requests-python`` package needs to -be installed in order for this to work. Additionally, the following needs to be -set in the master's configuration file: - -.. code-block:: bash - - backend: requests - -The following functions are available. - -list_input_endpoints --------------------- -List input endpoints associated with the deployment - -.. code-block:: bash - - salt-cloud -f list_input_endpoints my-azure service=myservice deployment=mydeployment - -show_input_endpoint -------------------- -Show an input endpoint associated with the deployment - -.. code-block:: bash - - salt-cloud -f show_input_endpoint my-azure service=myservice \ - deployment=mydeployment name=SSH - -add_input_endpoint ------------------- -Add an input endpoint to the deployment. Please note that there may be a delay -before the changes show up. The following options are available. - -service -~~~~~~~ -Required. The name of the hosted service which the VM belongs to. - -deployment -~~~~~~~~~~ -Required. The name of the deployment that the VM belongs to. If the VM was -created with Salt Cloud, the deployment name probably matches the VM name. - -role -~~~~ -Required. The name of the role that the VM belongs to. If the VM was created -with Salt Cloud, the role name probably matches the VM name. - -name -~~~~ -Required. The name of the input endpoint. This typically matches the port that -the endpoint is set to. For instance, port 22 would be called SSH. - -port -~~~~ -Required. The public (Internet-facing) port that is used for the endpoint. - -local_port -~~~~~~~~~~ -Optional. The private port on the VM itself that will be matched with the port. -This is typically the same as the ``port``. If this value is not specified, it -will be copied from ``port``. - -protocol -~~~~~~~~ -Required. Either ``tcp`` or ``udp``. - -enable_direct_server_return -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. If an internal load balancer exists in the account, it can be used -with a direct server return. The default value is ``False``. Please see the -following article for an explanation of this option. - -* `Load Balancing for Azure Infrastructure Services`__ - -.. __: https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-overview - -timeout_for_tcp_idle_connection -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. The default value is ``4``. Please see the following article for an -explanation of this option. - -* `Configurable Idle Timeout for Azure Load Balancer`__ - -.. __: https://azure.microsoft.com/en-us/blog/new-configurable-idle-timeout-for-azure-load-balancer/ - -CLI Example -~~~~~~~~~~~ -The following example illustrates adding an input endpoint. - -.. code-block:: bash - - salt-cloud -f add_input_endpoint my-azure service=myservice \ - deployment=mydeployment role=myrole name=HTTP local_port=80 \ - port=80 protocol=tcp enable_direct_server_return=False \ - timeout_for_tcp_idle_connection=4 - -update_input_endpoint ---------------------- -Updates the details for a specific input endpoint. All options from -``add_input_endpoint`` are supported. - -.. code-block:: bash - - salt-cloud -f update_input_endpoint my-azure service=myservice \ - deployment=mydeployment role=myrole name=HTTP local_port=80 \ - port=80 protocol=tcp enable_direct_server_return=False \ - timeout_for_tcp_idle_connection=4 - -delete_input_endpoint ---------------------- -Delete an input endpoint from the deployment. Please note that there may be a -delay before the changes show up. The following items are required. - -CLI Example -~~~~~~~~~~~ -The following example illustrates deleting an input endpoint. - -service -~~~~~~~ -The name of the hosted service which the VM belongs to. - -deployment -~~~~~~~~~~ -The name of the deployment that the VM belongs to. If the VM was created with -Salt Cloud, the deployment name probably matches the VM name. - -role -~~~~ -The name of the role that the VM belongs to. If the VM was created with Salt -Cloud, the role name probably matches the VM name. - -name -~~~~ -The name of the input endpoint. This typically matches the port that the -endpoint is set to. For instance, port 22 would be called SSH. - -.. code-block:: bash - - salt-cloud -f delete_input_endpoint my-azure service=myservice \ - deployment=mydeployment role=myrole name=HTTP - - -Managing Affinity Groups -======================== -.. versionadded:: 2015.8.0 - -Affinity groups allow you to group your Azure services to optimize performance. -All services and VMs within an affinity group will be located in the same -region. For more information on Affinity groups, see the following link: - -* `Create an Affinity Group in the Management Portal`__ - -.. __: https://msdn.microsoft.com/en-us/library/azure/jj156209.aspx - -The following functions are available. - -list_affinity_groups --------------------- -List input endpoints associated with the account - -.. code-block:: bash - - salt-cloud -f list_affinity_groups my-azure - -show_affinity_group -------------------- -Show an affinity group associated with the account - -.. code-block:: bash - - salt-cloud -f show_affinity_group my-azure service=myservice \ - deployment=mydeployment name=SSH - -create_affinity_group ---------------------- -Create a new affinity group. The following options are supported. - -name -~~~~ -Required. The name of the new affinity group. - -location -~~~~~~~~ -Required. The region in which the affinity group lives. - -label -~~~~~ -Required. A label describing the new affinity group. - -description -~~~~~~~~~~~ -Optional. A longer description of the affinity group. - -.. code-block:: bash - - salt-cloud -f create_affinity_group my-azure name=my_affinity_group \ - label=my-affinity-group location='West US' - -update_affinity_group ---------------------- -Update an affinity group's properties - -.. code-block:: bash - - salt-cloud -f update_affinity_group my-azure name=my_group label=my_group - -delete_affinity_group ---------------------- -Delete a specific affinity group associated with the account - -.. code-block:: bash - - salt-cloud -f delete_affinity_group my-azure name=my_affinity_group - - -Managing Blob Storage -===================== -.. versionadded:: 2015.8.0 - -Azure storage containers and their contents can be managed with Salt Cloud. This -is not as elegant as using one of the other available clients in Windows, but it -benefits Linux and Unix users, as there are fewer options available on those -platforms. - -Blob Storage Configuration --------------------------- -Blob storage must be configured differently than the standard Azure -configuration. Both a ``storage_account`` and a ``storage_key`` must be -specified either through the Azure provider configuration (in addition to the -other Azure configuration) or via the command line. - -.. code-block:: yaml - - storage_account: mystorage - storage_key: ffhj334fDSGFEGDFGFDewr34fwfsFSDFwe== - -storage_account -~~~~~~~~~~~~~~~ -This is one of the storage accounts that is available via the ``list_storage`` -function. - -storage_key -~~~~~~~~~~~ -Both a primary and a secondary ``storage_key`` can be obtained by running the -``show_storage_keys`` function. Either key may be used. - - -Blob Functions --------------- -The following functions are made available through Salt Cloud for managing -blog storage. - -make_blob_url -~~~~~~~~~~~~~ -Creates the URL to access a blob - -.. code-block:: bash - - salt-cloud -f make_blob_url my-azure container=mycontainer blob=myblob - -container -````````` -Name of the container. - -blob -```` -Name of the blob. - -account -``````` -Name of the storage account. If not specified, derives the host base -from the provider configuration. - -protocol -```````` -Protocol to use: 'http' or 'https'. If not specified, derives the host -base from the provider configuration. - -host_base -````````` -Live host base URL. If not specified, derives the host base from the -provider configuration. - - -list_storage_containers -~~~~~~~~~~~~~~~~~~~~~~~ -List containers associated with the storage account - -.. code-block:: bash - - salt-cloud -f list_storage_containers my-azure - - -create_storage_container -~~~~~~~~~~~~~~~~~~~~~~~~ -Create a storage container - -.. code-block:: bash - - salt-cloud -f create_storage_container my-azure name=mycontainer - -name -```` -Name of container to create. - -meta_name_values -```````````````` -Optional. A dict with name_value pairs to associate with the -container as metadata. Example:{'Category':'test'} - -blob_public_access -`````````````````` -Optional. Possible values include: container, blob - -fail_on_exist -````````````` -Specify whether to throw an exception when the container exists. - - -show_storage_container -~~~~~~~~~~~~~~~~~~~~~~ -Show a container associated with the storage account - -.. code-block:: bash - - salt-cloud -f show_storage_container my-azure name=myservice - -name -```` -Name of container to show. - - -show_storage_container_metadata -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Show a storage container's metadata - -.. code-block:: bash - - salt-cloud -f show_storage_container_metadata my-azure name=myservice - -name -```` -Name of container to show. - -lease_id -```````` -If specified, show_storage_container_metadata only succeeds if the -container's lease is active and matches this ID. - - -set_storage_container_metadata -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Set a storage container's metadata - -.. code-block:: bash - - salt-cloud -f set_storage_container my-azure name=mycontainer \ - x_ms_meta_name_values='{"my_name": "my_value"}' - -name -```` -Name of existing container. -meta_name_values -```````````````` -A dict containing name, value for metadata. -Example: {'category':'test'} -lease_id -```````` -If specified, set_storage_container_metadata only succeeds if the -container's lease is active and matches this ID. - - -show_storage_container_acl -~~~~~~~~~~~~~~~~~~~~~~~~~~ -Show a storage container's acl - -.. code-block:: bash - - salt-cloud -f show_storage_container_acl my-azure name=myservice - -name -```` -Name of existing container. - -lease_id -```````` -If specified, show_storage_container_acl only succeeds if the -container's lease is active and matches this ID. - - -set_storage_container_acl -~~~~~~~~~~~~~~~~~~~~~~~~~ -Set a storage container's acl - -.. code-block:: bash - - salt-cloud -f set_storage_container my-azure name=mycontainer - -name -```` -Name of existing container. - -signed_identifiers -`````````````````` -SignedIdentifiers instance - -blob_public_access -`````````````````` -Optional. Possible values include: container, blob - -lease_id -```````` -If specified, set_storage_container_acl only succeeds if the -container's lease is active and matches this ID. - - -delete_storage_container -~~~~~~~~~~~~~~~~~~~~~~~~ -Delete a container associated with the storage account - -.. code-block:: bash - - salt-cloud -f delete_storage_container my-azure name=mycontainer - -name -```` -Name of container to create. - -fail_not_exist -`````````````` -Specify whether to throw an exception when the container exists. - -lease_id -```````` -If specified, delete_storage_container only succeeds if the -container's lease is active and matches this ID. - - -lease_storage_container -~~~~~~~~~~~~~~~~~~~~~~~ -Lease a container associated with the storage account - -.. code-block:: bash - - salt-cloud -f lease_storage_container my-azure name=mycontainer - -name -```` -Name of container to create. - -lease_action -```````````` -Required. Possible values: acquire|renew|release|break|change - -lease_id -```````` -Required if the container has an active lease. - -lease_duration -`````````````` -Specifies the duration of the lease, in seconds, or negative one -(-1) for a lease that never expires. A non-infinite lease can be -between 15 and 60 seconds. A lease duration cannot be changed -using renew or change. For backwards compatibility, the default is -60, and the value is only used on an acquire operation. - -lease_break_period -`````````````````` -Optional. For a break operation, this is the proposed duration of -seconds that the lease should continue before it is broken, between -0 and 60 seconds. This break period is only used if it is shorter -than the time remaining on the lease. If longer, the time remaining -on the lease is used. A new lease will not be available before the -break period has expired, but the lease may be held for longer than -the break period. If this header does not appear with a break -operation, a fixed-duration lease breaks after the remaining lease -period elapses, and an infinite lease breaks immediately. - -proposed_lease_id -````````````````` -Optional for acquire, required for change. Proposed lease ID, in a -GUID string format. - - -list_blobs -~~~~~~~~~~ -List blobs associated with the container - -.. code-block:: bash - - salt-cloud -f list_blobs my-azure container=mycontainer - -container -````````` -The name of the storage container - -prefix -`````` -Optional. Filters the results to return only blobs whose names -begin with the specified prefix. - -marker -`````` -Optional. A string value that identifies the portion of the list -to be returned with the next list operation. The operation returns -a marker value within the response body if the list returned was -not complete. The marker value may then be used in a subsequent -call to request the next set of list items. The marker value is -opaque to the client. - -maxresults -`````````` -Optional. Specifies the maximum number of blobs to return, -including all BlobPrefix elements. If the request does not specify -maxresults or specifies a value greater than 5,000, the server will -return up to 5,000 items. Setting maxresults to a value less than -or equal to zero results in error response code 400 (Bad Request). - -include -``````` -Optional. Specifies one or more datasets to include in the -response. To specify more than one of these options on the URI, -you must separate each option with a comma. Valid values are:: - - snapshots: - Specifies that snapshots should be included in the - enumeration. Snapshots are listed from oldest to newest in - the response. - metadata: - Specifies that blob metadata be returned in the response. - uncommittedblobs: - Specifies that blobs for which blocks have been uploaded, - but which have not been committed using Put Block List - (REST API), be included in the response. - copy: - Version 2012-02-12 and newer. Specifies that metadata - related to any current or previous Copy Blob operation - should be included in the response. - -delimiter -````````` -Optional. When the request includes this parameter, the operation -returns a BlobPrefix element in the response body that acts as a -placeholder for all blobs whose names begin with the same -substring up to the appearance of the delimiter character. The -delimiter may be a single character or a string. - - -show_blob_service_properties -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Show a blob's service properties - -.. code-block:: bash - - salt-cloud -f show_blob_service_properties my-azure - - -set_blob_service_properties -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Sets the properties of a storage account's Blob service, including -Windows Azure Storage Analytics. You can also use this operation to -set the default request version for all incoming requests that do not -have a version specified. - -.. code-block:: bash - - salt-cloud -f set_blob_service_properties my-azure - -properties -`````````` -a StorageServiceProperties object. - -timeout -``````` -Optional. The timeout parameter is expressed in seconds. - - -show_blob_properties -~~~~~~~~~~~~~~~~~~~~ -Returns all user-defined metadata, standard HTTP properties, and -system properties for the blob. - -.. code-block:: bash - - salt-cloud -f show_blob_properties my-azure container=mycontainer blob=myblob - -container -````````` -Name of existing container. - -blob -```` -Name of existing blob. - -lease_id -```````` -Required if the blob has an active lease. - - -set_blob_properties -~~~~~~~~~~~~~~~~~~~ -Set a blob's properties - -.. code-block:: bash - - salt-cloud -f set_blob_properties my-azure - -container -````````` -Name of existing container. - -blob -```` -Name of existing blob. - -blob_cache_control -`````````````````` -Optional. Modifies the cache control string for the blob. - -blob_content_type -````````````````` -Optional. Sets the blob's content type. - -blob_content_md5 -```````````````` -Optional. Sets the blob's MD5 hash. - -blob_content_encoding -````````````````````` -Optional. Sets the blob's content encoding. - -blob_content_language -````````````````````` -Optional. Sets the blob's content language. - -lease_id -```````` -Required if the blob has an active lease. - -blob_content_disposition -```````````````````````` -Optional. Sets the blob's Content-Disposition header. -The Content-Disposition response header field conveys additional -information about how to process the response payload, and also can -be used to attach additional metadata. For example, if set to -attachment, it indicates that the user-agent should not display the -response, but instead show a Save As dialog with a filename other -than the blob name specified. - - -put_blob -~~~~~~~~ -Upload a blob - -.. code-block:: bash - - salt-cloud -f put_blob my-azure container=base name=top.sls blob_path=/srv/salt/top.sls - salt-cloud -f put_blob my-azure container=base name=content.txt blob_content='Some content' - -container -````````` -Name of existing container. - -name -```` -Name of existing blob. - -blob_path -````````` -The path on the local machine of the file to upload as a blob. Either -this or blob_content must be specified. - -blob_content -```````````` -The actual content to be uploaded as a blob. Either this or blob_path -must me specified. - -cache_control -````````````` -Optional. The Blob service stores this value but does not use or -modify it. - -content_language -```````````````` -Optional. Specifies the natural languages used by this resource. - -content_md5 -``````````` -Optional. An MD5 hash of the blob content. This hash is used to -verify the integrity of the blob during transport. When this header -is specified, the storage service checks the hash that has arrived -with the one that was sent. If the two hashes do not match, the -operation will fail with error code 400 (Bad Request). - -blob_content_type -````````````````` -Optional. Set the blob's content type. - -blob_content_encoding -````````````````````` -Optional. Set the blob's content encoding. - -blob_content_language -````````````````````` -Optional. Set the blob's content language. - -blob_content_md5 -```````````````` -Optional. Set the blob's MD5 hash. - -blob_cache_control -`````````````````` -Optional. Sets the blob's cache control. - -meta_name_values -```````````````` -A dict containing name, value for metadata. - -lease_id -```````` -Required if the blob has an active lease. - - -get_blob -~~~~~~~~ -Download a blob - -.. code-block:: bash - - salt-cloud -f get_blob my-azure container=base name=top.sls local_path=/srv/salt/top.sls - salt-cloud -f get_blob my-azure container=base name=content.txt return_content=True - -container -````````` -Name of existing container. - -name -```` -Name of existing blob. - -local_path -`````````` -The path on the local machine to download the blob to. Either this or -return_content must be specified. - -return_content -`````````````` -Whether or not to return the content directly from the blob. If -specified, must be True or False. Either this or the local_path must -be specified. - -snapshot -```````` -Optional. The snapshot parameter is an opaque DateTime value that, -when present, specifies the blob snapshot to retrieve. - -lease_id -```````` -Required if the blob has an active lease. - -progress_callback -````````````````` -callback for progress with signature function(current, total) where -current is the number of bytes transferred so far, and total is the -size of the blob. - -max_connections -``````````````` -Maximum number of parallel connections to use when the blob size -exceeds 64MB. -Set to 1 to download the blob chunks sequentially. -Set to 2 or more to download the blob chunks in parallel. This uses -more system resources but will download faster. - -max_retries -``````````` -Number of times to retry download of blob chunk if an error occurs. - -retry_wait -`````````` -Sleep time in secs between retries. diff --git a/doc/topics/cloud/azurearm.rst b/doc/topics/cloud/azurearm.rst deleted file mode 100644 index cc9b2122819..00000000000 --- a/doc/topics/cloud/azurearm.rst +++ /dev/null @@ -1,486 +0,0 @@ -============================== -Getting Started With Azure ARM -============================== - -.. versionadded:: 2016.11.0 - -.. warning:: - - This cloud provider will be removed from Salt in version 3007 in favor of - the `saltext.azurerm Salt Extension - `_ - -Azure is a cloud service by Microsoft providing virtual machines, SQL services, -media services, and more. Azure ARM (aka, the Azure Resource Manager) is a next -generation version of the Azure portal and API. This document describes how to -use Salt Cloud to create a virtual machine on Azure ARM, with Salt installed. - -More information about Azure is located at `http://www.windowsazure.com/ -`_. - - -Dependencies -============ -* `azure `_ >= 2.0.0rc6 -* `azure-common `_ >= 1.1.4 -* `azure-mgmt `_ >= 0.30.0rc6 -* `azure-mgmt-compute `_ >= 0.33.0 -* `azure-mgmt-network `_ >= 0.30.0rc6 -* `azure-mgmt-resource `_ >= 0.30.0 -* `azure-mgmt-storage `_ >= 0.30.0rc6 -* `azure-mgmt-web `_ >= 0.30.0rc6 -* `azure-storage `_ >= 0.32.0 -* `msrestazure `_ >= 0.4.21 -* A Microsoft Azure account -* `Salt `_ - - -Installation Tips -================= -Because the ``azure`` library requires the ``cryptography`` library, which is -compiled on-the-fly by ``pip``, you may need to install the development tools -for your operating system. - -Before you install ``azure`` with ``pip``, you should make sure that the -required libraries are installed. - -Debian ------- -For Debian and Ubuntu, the following command will ensure that the required -dependencies are installed: - -.. code-block:: bash - - sudo apt-get install build-essential libssl-dev libffi-dev python-dev - -Red Hat -------- -For Fedora and RHEL-derivatives, the following command will ensure that the -required dependencies are installed: - -.. code-block:: bash - - sudo yum install gcc libffi-devel python-devel openssl-devel - - -Configuration -============= - -Set up the provider config at ``/etc/salt/cloud.providers.d/azurearm.conf``: - -.. code-block:: yaml - - # Note: This example is for /etc/salt/cloud.providers.d/azurearm.conf - - my-azurearm-config: - driver: azurearm - master: salt.example.com - subscription_id: 01234567-890a-bcde-f012-34567890abdc - - # https://apps.dev.microsoft.com/#/appList - username: @.onmicrosoft.com - password: verybadpass - location: westus - resource_group: my_rg - - # Optional - network_resource_group: my_net_rg - cleanup_disks: True - cleanup_vhds: True - cleanup_data_disks: True - cleanup_interfaces: True - custom_data: 'This is custom data' - expire_publisher_cache: 604800 # 7 days - expire_offer_cache: 518400 # 6 days - expire_sku_cache: 432000 # 5 days - expire_version_cache: 345600 # 4 days - expire_group_cache: 14400 # 4 hours - expire_interface_cache: 3600 # 1 hour - expire_network_cache: 3600 # 1 hour - -Cloud Profiles -============== -Set up an initial profile at ``/etc/salt/cloud.profiles``: - -.. code-block:: yaml - - azure-ubuntu-pass: - provider: my-azure-config - image: Canonical|UbuntuServer|14.04.5-LTS|14.04.201612050 - size: Standard_D1_v2 - location: eastus - ssh_username: azureuser - ssh_password: verybadpass - - azure-ubuntu-key: - provider: my-azure-config - image: Canonical|UbuntuServer|14.04.5-LTS|14.04.201612050 - size: Standard_D1_v2 - location: eastus - ssh_username: azureuser - ssh_publickeyfile: /path/to/ssh_public_key.pub - - azure-win2012: - provider: my-azure-config - image: MicrosoftWindowsServer|WindowsServer|2012-R2-Datacenter|latest - size: Standard_D1_v2 - location: westus - win_username: azureuser - win_password: verybadpass - -These options are described in more detail below. Once configured, the profile -can be realized with a salt command: - -.. code-block:: bash - - salt-cloud -p azure-ubuntu newinstance - -This will create an salt minion instance named ``newinstance`` in Azure. If -the command was executed on the salt-master, its Salt key will automatically -be signed on the master. - -Once the instance has been created with salt-minion installed, connectivity to -it can be verified with Salt: - -.. code-block:: bash - - salt newinstance test.version - - -Profile Options -=============== -The following options are currently available for Azure ARM. - -provider --------- -The name of the provider as configured in -`/etc/salt/cloud.providers.d/azure.conf`. - -image ------ -Required. The name of the image to use to create a VM. Available images can be -viewed using the following command: - -.. code-block:: bash - - salt-cloud --list-images my-azure-config - -As you will see in ``--list-images``, image names are comprised of the following -fields, separated by the pipe (``|``) character: - -.. code-block:: yaml - - publisher: For example, Canonical or MicrosoftWindowsServer - offer: For example, UbuntuServer or WindowsServer - sku: Such as 14.04.5-LTS or 2012-R2-Datacenter - version: Such as 14.04.201612050 or latest - -It is possible to specify the URL or resource ID path of a custom image that you -have access to, such as: - -.. code-block:: yaml - - https://.blob.core.windows.net/system/Microsoft.Compute/Images//template-osDisk.01234567-890a-bcdef0123-4567890abcde.vhd - -or: - -.. code-block:: yaml - - /subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/resourceGroups/myRG/providers/Microsoft.Compute/images/myImage - -size ----- -Required. The name of the size to use to create a VM. Available sizes can be -viewed using the following command: - -.. code-block:: bash - - salt-cloud --list-sizes my-azure-config - -location --------- -Required. The name of the location to create a VM in. Available locations can -be viewed using the following command: - -.. code-block:: bash - - salt-cloud --list-locations my-azure-config - -ssh_username ------------- -Required for Linux. The admin user to add on the instance. It is also used to log -into the newly-created VM to install Salt. - -ssh_keyfile ------------ -Required if using SSH key authentication. The path on the Salt master to the SSH private -key used during the minion bootstrap process. - -ssh_publickeyfile ------------------ -Use either ``ssh_publickeyfile`` or ``ssh_password``. The path on the Salt master to the -SSH public key which will be pushed to the Linux VM. - -ssh_password ------------- -Use either ``ssh_publickeyfile`` or ``ssh_password``. The password for the admin user on -the newly-created Linux virtual machine. - -win_username ------------- -Required for Windows. The user to use to log into the newly-created Windows VM -to install Salt. - -win_password ------------- -Required for Windows. The password to use to log into the newly-created Windows -VM to install Salt. - -win_installer -------------- -Required for Windows. The path to the Salt installer to be uploaded. - -resource_group --------------- -Required. The resource group that all VM resources (VM, network interfaces, -etc) will be created in. - -network_resource_group ----------------------- -Optional. If specified, then the VM will be connected to the virtual network -in this resource group, rather than the parent resource group of the instance. -The VM interfaces and IPs will remain in the configured ``resource_group`` with -the VM. - -network -------- -Required. The virtual network that the VM will be spun up in. - -subnet ------- -Optional. The subnet inside the virtual network that the VM will be spun up in. -Default is ``default``. - -allocate_public_ip ------------------- -Optional. Default is ``False``. If set to ``True``, a public IP will -be created and assigned to the VM. - -load_balancer -------------- -Optional. The load-balancer for the VM's network interface to join. If -specified the backend_pool option need to be set. - -backend_pool ------------- -Optional. Required if the load_balancer option is set. The load-balancer's -Backend Pool the VM's network interface will join. - -iface_name ----------- -Optional. The name to apply to the VM's network interface. If not supplied, the -value will be set to ``-iface0``. - -dns_servers ------------ -Optional. A **list** of the DNS servers to configure for the network interface -(will be set on the VM by the DHCP of the VNET). - -.. code-block:: yaml - - my-azurearm-profile: - provider: azurearm-provider - network: mynetwork - dns_servers: - - 10.1.1.4 - - 10.1.1.5 - -availability_set ----------------- -Optional. If set, the VM will be added to the specified availability set. - -volumes -------- - -Optional. A list of dictionaries describing data disks to attach to the -instance can be specified using this setting. The data disk dictionaries are -passed entirely to the `Azure DataDisk object -`_, -so ad-hoc options can be handled as long as they are valid properties of the -object. - -.. code-block:: yaml - - volumes: - - disk_size_gb: 50 - caching: ReadWrite - - disk_size_gb: 100 - caching: ReadWrite - managed_disk: - storage_account_type: Standard_LRS - -cleanup_disks -------------- -Optional. Default is ``False``. If set to ``True``, disks will be cleaned up -when the VM that they belong to is deleted. - -cleanup_vhds ------------- -Optional. Default is ``False``. If set to ``True``, VHDs will be cleaned up -when the VM and disk that they belong to are deleted. Requires ``cleanup_disks`` -to be set to ``True``. - -cleanup_data_disks ------------------- -Optional. Default is ``False``. If set to ``True``, data disks (non-root -volumes) will be cleaned up whtn the VM that they are attached to is deleted. -Requires ``cleanup_disks`` to be set to ``True``. - -cleanup_interfaces ------------------- -Optional. Default is ``False``. Normally when a VM is deleted, its associated -interfaces and IPs are retained. This is useful if you expect the deleted VM -to be recreated with the same name and network settings. If you would like -interfaces and IPs to be deleted when their associated VM is deleted, set this -to ``True``. - -userdata --------- -Optional. Any custom cloud data that needs to be specified. How this data is -used depends on the operating system and image that is used. For instance, -Linux images that use ``cloud-init`` will import this data for use with that -program. Some Windows images will create a file with a copy of this data, and -others will ignore it. If a Windows image creates a file, then the location -will depend upon the version of Windows. This will be ignored if the -``userdata_file`` is specified. - -userdata_file -------------- -Optional. The path to a file to be read and submitted to Azure as user data. -How this is used depends on the operating system that is being deployed. If -used, any ``userdata`` setting will be ignored. - -userdata_sendkeys ------------------ -Optional. Set to ``True`` in order to generate salt minion keys and provide -them as variables to the userdata script when running it through the template -renderer. The keys can be referenced as ``{{opts['priv_key']}}`` and -``{{opts['pub_key']}}``. - -userdata_template ------------------ -Optional. Enter the renderer, such as ``jinja``, to be used for the userdata -script template. - -wait_for_ip_timeout -------------------- -Optional. Default is ``600``. When waiting for a VM to be created, Salt Cloud -will attempt to connect to the VM's IP address until it starts responding. This -setting specifies the maximum time to wait for a response. - -wait_for_ip_interval --------------------- -Optional. Default is ``10``. How long to wait between attempts to connect to -the VM's IP. - -wait_for_ip_interval_multiplier -------------------------------- -Optional. Default is ``1``. Increase the interval by this multiplier after -each request; helps with throttling. - -expire_publisher_cache ----------------------- -Optional. Default is ``604800``. When fetching image data using -``--list-images``, a number of web calls need to be made to the Azure ARM API. -This is normally very fast when performed using a VM that exists inside Azure -itself, but can be very slow when made from an external connection. - -By default, the publisher data will be cached, and only updated every ``604800`` -seconds (7 days). If you need the publisher cache to be updated at a different -frequency, change this setting. Setting it to ``0`` will turn off the publisher -cache. - -expire_offer_cache ------------------- -Optional. Default is ``518400``. See ``expire_publisher_cache`` for details on -why this exists. - -By default, the offer data will be cached, and only updated every ``518400`` -seconds (6 days). If you need the offer cache to be updated at a different -frequency, change this setting. Setting it to ``0`` will turn off the publiser -cache. - -expire_sku_cache ----------------- -Optional. Default is ``432000``. See ``expire_publisher_cache`` for details on -why this exists. - -By default, the sku data will be cached, and only updated every ``432000`` -seconds (5 days). If you need the sku cache to be updated at a different -frequency, change this setting. Setting it to ``0`` will turn off the sku -cache. - -expire_version_cache --------------------- -Optional. Default is ``345600``. See ``expire_publisher_cache`` for details on -why this exists. - -By default, the version data will be cached, and only updated every ``345600`` -seconds (4 days). If you need the version cache to be updated at a different -frequency, change this setting. Setting it to ``0`` will turn off the version -cache. - -expire_group_cache ------------------- -Optional. Default is ``14400``. See ``expire_publisher_cache`` for details on -why this exists. - -By default, the resource group data will be cached, and only updated every -``14400`` seconds (4 hours). If you need the resource group cache to be updated -at a different frequency, change this setting. Setting it to ``0`` will turn -off the resource group cache. - -expire_interface_cache ----------------------- -Optional. Default is ``3600``. See ``expire_publisher_cache`` for details on -why this exists. - -By default, the interface data will be cached, and only updated every ``3600`` -seconds (1 hour). If you need the interface cache to be updated at a different -frequency, change this setting. Setting it to ``0`` will turn off the interface -cache. - -expire_network_cache --------------------- -Optional. Default is ``3600``. See ``expire_publisher_cache`` for details on -why this exists. - -By default, the network data will be cached, and only updated every ``3600`` -seconds (1 hour). If you need the network cache to be updated at a different -frequency, change this setting. Setting it to ``0`` will turn off the network -cache. - - -Other Options -============= -Other options relevant to Azure ARM. - -storage_account ---------------- -Required for actions involving an Azure storage account. - -storage_key ------------ -Required for actions involving an Azure storage account. - - -Show Instance -============= -This action is a thin wrapper around ``--full-query``, which displays details on -a single instance only. In an environment with several machines, this will save -a user from having to sort through all instance data, just to examine a single -instance. - -.. code-block:: bash - - salt-cloud -a show_instance myinstance diff --git a/salt/cloud/clouds/azurearm.py b/salt/cloud/clouds/azurearm.py deleted file mode 100644 index de93ac18a73..00000000000 --- a/salt/cloud/clouds/azurearm.py +++ /dev/null @@ -1,2043 +0,0 @@ -""" -Azure ARM Cloud Module -====================== - -.. versionadded:: 2016.11.0 - -.. versionchanged:: 2019.2.0 - -The Azure ARM cloud module is used to control access to Microsoft Azure Resource Manager - -.. warning:: - - This cloud provider will be removed from Salt in version 3007 in favor of - the `saltext.azurerm Salt Extension - `_ - -:maintainer: -:depends: - * `azure `_ >= 2.0.0rc6 - * `azure-common `_ >= 1.1.4 - * `azure-mgmt `_ >= 0.30.0rc6 - * `azure-mgmt-compute `_ >= 0.33.0 - * `azure-mgmt-network `_ >= 0.30.0rc6 - * `azure-mgmt-resource `_ >= 0.30.0 - * `azure-mgmt-storage `_ >= 0.30.0rc6 - * `azure-mgmt-web `_ >= 0.30.0rc6 - * `azure-storage `_ >= 0.32.0 - * `msrestazure `_ >= 0.4.21 -:configuration: - Required provider parameters: - - if using username and password: - * ``subscription_id`` - * ``username`` - * ``password`` - - if using a service principal: - * ``subscription_id`` - * ``tenant`` - * ``client_id`` - * ``secret`` - - if using Managed Service Identity authentication: - * ``subscription_id`` - - Optional provider parameters: - - **cloud_environment**: Used to point the cloud driver to different API endpoints, such as Azure GovCloud. Possible values: - * ``AZURE_PUBLIC_CLOUD`` (default) - * ``AZURE_CHINA_CLOUD`` - * ``AZURE_US_GOV_CLOUD`` - * ``AZURE_GERMAN_CLOUD`` - * HTTP base URL for a custom endpoint, such as Azure Stack. The ``/metadata/endpoints`` path will be added to the URL. - - **userdata** and **userdata_file**: - Azure Resource Manager uses a separate VirtualMachineExtension object to pass userdata scripts to the virtual - machine. Arbitrary shell commands can be passed via the ``userdata`` parameter, or via a file local to the Salt - Cloud system using the ``userdata_file`` parameter. Note that the local file is not treated as a script by the - extension, so "one-liners" probably work best. If greater functionality is desired, a web-hosted script file can - be specified via ``userdata_file: https://raw.githubusercontent.com/account/repo/master/azure-script.py``, which - will be executed on the system after VM creation. For Windows systems, script files ending in ``.ps1`` will be - executed with ``powershell.exe``. The ``userdata`` parameter takes precedence over the ``userdata_file`` parameter - when creating the custom script extension. - - **win_installer**: - This parameter, which holds the local path to the Salt Minion installer package, is used to determine if the - virtual machine type will be "Windows". Only set this parameter on profiles which install Windows operating systems. - - -Example ``/etc/salt/cloud.providers`` or -``/etc/salt/cloud.providers.d/azure.conf`` configuration: - -.. code-block:: yaml - - my-azure-config with username and password: - driver: azurearm - subscription_id: 3287abc8-f98a-c678-3bde-326766fd3617 - username: larry - password: 123pass - - Or my-azure-config with service principal: - driver: azurearm - subscription_id: 3287abc8-f98a-c678-3bde-326766fd3617 - tenant: ABCDEFAB-1234-ABCD-1234-ABCDEFABCDEF - client_id: ABCDEFAB-1234-ABCD-1234-ABCDEFABCDEF - secret: XXXXXXXXXXXXXXXXXXXXXXXX - cloud_environment: AZURE_US_GOV_CLOUD - - The Service Principal can be created with the new Azure CLI (https://github.com/Azure/azure-cli) with: - az ad sp create-for-rbac -n "http://" --role --scopes - For example, this creates a service principal with 'owner' role for the whole subscription: - az ad sp create-for-rbac -n "http://mysaltapp" --role owner --scopes /subscriptions/3287abc8-f98a-c678-3bde-326766fd3617 - - *Note: review the details of Service Principals. Owner role is more than you normally need, and you can restrict - scope to a resource group or individual resources. -""" - -import importlib -import logging -import os -import os.path -import pprint -import string -import time -from functools import wraps -from multiprocessing import cpu_count -from multiprocessing.pool import ThreadPool - -import salt.cache -import salt.config as config -import salt.loader -import salt.utils.azurearm -import salt.utils.cloud -import salt.utils.files -import salt.utils.stringutils -import salt.utils.yaml -import salt.version -from salt.exceptions import ( - SaltCloudConfigError, - SaltCloudExecutionFailure, - SaltCloudExecutionTimeout, - SaltCloudSystemExit, -) - -HAS_LIBS = False -try: - import azure.mgmt.compute.models as compute_models - import azure.mgmt.network.models as network_models - from azure.storage.blob.blockblobservice import BlockBlobService - from msrestazure.azure_exceptions import CloudError - - HAS_LIBS = True -except ImportError: - pass - -__virtualname__ = "azurearm" - -log = logging.getLogger(__name__) - - -def __virtual__(): - """ - Check for Azure configurations. - """ - if get_configured_provider() is False: - return False - - if get_dependencies() is False: - return ( - False, - "The following dependencies are required to use the AzureARM driver: " - "Microsoft Azure SDK for Python >= 2.0rc6, " - "Microsoft Azure Storage SDK for Python >= 0.32, " - "MS REST Azure (msrestazure) >= 0.4", - ) - - return __virtualname__ - - -def _get_active_provider_name(): - try: - return __active_provider_name__.value() - except AttributeError: - return __active_provider_name__ - - -def _deprecation_message(function): - """ - Decorator wrapper to warn about msazure deprecation - """ - - @wraps(function) - def wrapped(*args, **kwargs): - salt.utils.versions.warn_until( - "Chlorine", - "This cloud provider will be removed from Salt in version 3007 due to " - "the deprecation of the 'Classic' API for Azure. Please migrate to " - "Azure Resource Manager by March 1, 2023 " - "(https://docs.microsoft.com/en-us/azure/virtual-machines/classic-vm-deprecation)", - category=FutureWarning, - ) - ret = function(*args, **salt.utils.args.clean_kwargs(**kwargs)) - return ret - - return wrapped - - -@_deprecation_message -def get_api_versions(call=None, kwargs=None): # pylint: disable=unused-argument - """ - Get a resource type api versions - """ - if kwargs is None: - kwargs = {} - - if "resource_provider" not in kwargs: - raise SaltCloudSystemExit("A resource_provider must be specified") - - if "resource_type" not in kwargs: - raise SaltCloudSystemExit("A resource_type must be specified") - - api_versions = [] - - try: - resconn = get_conn(client_type="resource") - provider_query = resconn.providers.get( - resource_provider_namespace=kwargs["resource_provider"] - ) - - for resource in provider_query.resource_types: - if str(resource.resource_type) == kwargs["resource_type"]: - resource_dict = resource.as_dict() - api_versions = resource_dict["api_versions"] - except CloudError as exc: - salt.utils.azurearm.log_cloud_error("resource", exc.message) - - return api_versions - - -@_deprecation_message -def get_resource_by_id(resource_id, api_version, extract_value=None): - """ - Get an AzureARM resource by id - """ - ret = {} - - try: - resconn = get_conn(client_type="resource") - resource_query = resconn.resources.get_by_id( - resource_id=resource_id, api_version=api_version - ) - resource_dict = resource_query.as_dict() - if extract_value is not None: - ret = resource_dict[extract_value] - else: - ret = resource_dict - except CloudError as exc: - salt.utils.azurearm.log_cloud_error("resource", exc.message) - ret = {"Error": exc.message} - - return ret - - -def get_configured_provider(): - """ - Return the first configured provider instance. - """ - key_combos = [ - ("subscription_id", "tenant", "client_id", "secret"), - ("subscription_id", "username", "password"), - ("subscription_id",), - ] - - for combo in key_combos: - provider = config.is_provider_configured( - __opts__, - _get_active_provider_name() or __virtualname__, - combo, - ) - - if provider: - return provider - - return provider - - -@_deprecation_message -def get_dependencies(): - """ - Warn if dependencies aren't met. - """ - return config.check_driver_dependencies(__virtualname__, {"azurearm": HAS_LIBS}) - - -@_deprecation_message -def get_conn(client_type): - """ - Return a connection object for a client type. - """ - conn_kwargs = {} - - conn_kwargs["subscription_id"] = salt.utils.stringutils.to_str( - config.get_cloud_config_value( - "subscription_id", get_configured_provider(), __opts__, search_global=False - ) - ) - - cloud_env = config.get_cloud_config_value( - "cloud_environment", get_configured_provider(), __opts__, search_global=False - ) - - if cloud_env is not None: - conn_kwargs["cloud_environment"] = cloud_env - - tenant = config.get_cloud_config_value( - "tenant", get_configured_provider(), __opts__, search_global=False - ) - - if tenant is not None: - client_id = config.get_cloud_config_value( - "client_id", get_configured_provider(), __opts__, search_global=False - ) - secret = config.get_cloud_config_value( - "secret", get_configured_provider(), __opts__, search_global=False - ) - conn_kwargs.update({"client_id": client_id, "secret": secret, "tenant": tenant}) - - username = config.get_cloud_config_value( - "username", get_configured_provider(), __opts__, search_global=False - ) - - if username: - password = config.get_cloud_config_value( - "password", get_configured_provider(), __opts__, search_global=False - ) - conn_kwargs.update({"username": username, "password": password}) - - client = salt.utils.azurearm.get_client(client_type=client_type, **conn_kwargs) - - return client - - -@_deprecation_message -def get_location(call=None, kwargs=None): # pylint: disable=unused-argument - """ - Return the location that is configured for this provider - """ - if not kwargs: - kwargs = {} - vm_dict = get_configured_provider() - vm_dict.update(kwargs) - return config.get_cloud_config_value( - "location", vm_dict, __opts__, search_global=False - ) - - -@_deprecation_message -def avail_locations(call=None): - """ - Return a dict of all available regions. - """ - if call == "action": - raise SaltCloudSystemExit( - "The avail_locations function must be called with " - "-f or --function, or with the --list-locations option" - ) - - ret = {} - ret["locations"] = [] - - try: - resconn = get_conn(client_type="resource") - provider_query = resconn.providers.get( - resource_provider_namespace="Microsoft.Compute" - ) - locations = [] - for resource in provider_query.resource_types: - if str(resource.resource_type) == "virtualMachines": - resource_dict = resource.as_dict() - locations = resource_dict["locations"] - for location in locations: - lowercase = location.lower().replace(" ", "") - ret["locations"].append(lowercase) - except CloudError as exc: - salt.utils.azurearm.log_cloud_error("resource", exc.message) - ret = {"Error": exc.message} - - return ret - - -@_deprecation_message -def avail_images(call=None): - """ - Return a dict of all available images on the provider - """ - if call == "action": - raise SaltCloudSystemExit( - "The avail_images function must be called with " - "-f or --function, or with the --list-images option" - ) - compconn = get_conn(client_type="compute") - region = get_location() - publishers = [] - ret = {} - - def _get_publisher_images(publisher): - """ - Get all images from a specific publisher - """ - data = {} - try: - offers = compconn.virtual_machine_images.list_offers( - location=region, - publisher_name=publisher, - ) - for offer_obj in offers: - offer = offer_obj.as_dict() - skus = compconn.virtual_machine_images.list_skus( - location=region, - publisher_name=publisher, - offer=offer["name"], - ) - for sku_obj in skus: - sku = sku_obj.as_dict() - results = compconn.virtual_machine_images.list( - location=region, - publisher_name=publisher, - offer=offer["name"], - skus=sku["name"], - ) - for version_obj in results: - version = version_obj.as_dict() - name = "|".join( - ( - publisher, - offer["name"], - sku["name"], - version["name"], - ) - ) - data[name] = { - "publisher": publisher, - "offer": offer["name"], - "sku": sku["name"], - "version": version["name"], - } - except CloudError as exc: - salt.utils.azurearm.log_cloud_error("compute", exc.message) - data = {publisher: exc.message} - - return data - - try: - publishers_query = compconn.virtual_machine_images.list_publishers( - location=region - ) - for publisher_obj in publishers_query: - publisher = publisher_obj.as_dict() - publishers.append(publisher["name"]) - except CloudError as exc: - salt.utils.azurearm.log_cloud_error("compute", exc.message) - - pool = ThreadPool(cpu_count() * 6) - results = pool.map_async(_get_publisher_images, publishers) - results.wait() - - ret = {k: v for result in results.get() for k, v in result.items()} - - return ret - - -@_deprecation_message -def avail_sizes(call=None): - """ - Return a list of sizes available from the provider - """ - if call == "action": - raise SaltCloudSystemExit( - "The avail_sizes function must be called with " - "-f or --function, or with the --list-sizes option" - ) - - compconn = get_conn(client_type="compute") - - ret = {} - location = get_location() - - try: - sizes = compconn.virtual_machine_sizes.list(location=location) - for size_obj in sizes: - size = size_obj.as_dict() - ret[size["name"]] = size - except CloudError as exc: - salt.utils.azurearm.log_cloud_error("compute", exc.message) - ret = {"Error": exc.message} - - return ret - - -@_deprecation_message -def list_nodes(call=None): - """ - List VMs on this Azure account - """ - if call == "action": - raise SaltCloudSystemExit( - "The list_nodes function must be called with -f or --function." - ) - - ret = {} - - nodes = list_nodes_full() - for node in nodes: - ret[node] = {"name": node} - for prop in ("id", "image", "size", "state", "private_ips", "public_ips"): - ret[node][prop] = nodes[node].get(prop) - return ret - - -@_deprecation_message -def list_nodes_full(call=None): - """ - List all VMs on the subscription with full information - """ - if call == "action": - raise SaltCloudSystemExit( - "The list_nodes_full function must be called with -f or --function." - ) - - netapi_versions = get_api_versions( - kwargs={ - "resource_provider": "Microsoft.Network", - "resource_type": "networkInterfaces", - } - ) - netapi_version = netapi_versions[0] - compconn = get_conn(client_type="compute") - - ret = {} - - def _get_node_info(node): - """ - Get node info. - """ - node_ret = {} - node["id"] = node["vm_id"] - node["size"] = node["hardware_profile"]["vm_size"] - node["state"] = node["provisioning_state"] - node["public_ips"] = [] - node["private_ips"] = [] - node_ret[node["name"]] = node - try: - image_ref = node["storage_profile"]["image_reference"] - node["image"] = "|".join( - [ - image_ref["publisher"], - image_ref["offer"], - image_ref["sku"], - image_ref["version"], - ] - ) - except (TypeError, KeyError): - try: - node["image"] = node["storage_profile"]["os_disk"]["image"]["uri"] - except (TypeError, KeyError): - node["image"] = ( - node.get("storage_profile", {}).get("image_reference", {}).get("id") - ) - try: - netifaces = node["network_profile"]["network_interfaces"] - for index, netiface in enumerate(netifaces): - netiface_name = get_resource_by_id( - netiface["id"], netapi_version, "name" - ) - netiface, pubips, privips = _get_network_interface( - netiface_name, node["resource_group"] - ) - node["network_profile"]["network_interfaces"][index].update(netiface) - node["public_ips"].extend(pubips) - node["private_ips"].extend(privips) - except Exception: # pylint: disable=broad-except - pass - - node_ret[node["name"]] = node - - return node_ret - - for group in list_resource_groups(): - nodes = [] - nodes_query = compconn.virtual_machines.list(resource_group_name=group) - for node_obj in nodes_query: - node = node_obj.as_dict() - node["resource_group"] = group - nodes.append(node) - - pool = ThreadPool(cpu_count() * 6) - results = pool.map_async(_get_node_info, nodes) - results.wait() - - group_ret = {k: v for result in results.get() for k, v in result.items()} - ret.update(group_ret) - - return ret - - -@_deprecation_message -def list_resource_groups(call=None): - """ - List resource groups associated with the subscription - """ - if call == "action": - raise SaltCloudSystemExit( - "The list_hosted_services function must be called with -f or --function" - ) - - resconn = get_conn(client_type="resource") - ret = {} - try: - groups = resconn.resource_groups.list() - - for group_obj in groups: - group = group_obj.as_dict() - ret[group["name"]] = group - except CloudError as exc: - salt.utils.azurearm.log_cloud_error("resource", exc.message) - ret = {"Error": exc.message} - - return ret - - -@_deprecation_message -def show_instance(name, call=None): - """ - Show the details from AzureARM concerning an instance - """ - if call != "action": - raise SaltCloudSystemExit( - "The show_instance action must be called with -a or --action." - ) - try: - node = list_nodes_full("function")[name] - except KeyError: - log.debug("Failed to get data for node '%s'", name) - node = {} - - __utils__["cloud.cache_node"](node, _get_active_provider_name(), __opts__) - - return node - - -@_deprecation_message -def delete_interface(call=None, kwargs=None): # pylint: disable=unused-argument - """ - Delete a network interface. - """ - if kwargs is None: - kwargs = {} - - netconn = get_conn(client_type="network") - - if kwargs.get("resource_group") is None: - kwargs["resource_group"] = config.get_cloud_config_value( - "resource_group", {}, __opts__, search_global=True - ) - - ips = [] - iface = netconn.network_interfaces.get( - kwargs["resource_group"], - kwargs["iface_name"], - ) - iface_name = iface.name - for ip_ in iface.ip_configurations: - ips.append(ip_.name) - - poller = netconn.network_interfaces.delete( - kwargs["resource_group"], - kwargs["iface_name"], - ) - poller.wait() - - for ip_ in ips: - poller = netconn.public_ip_addresses.delete(kwargs["resource_group"], ip_) - poller.wait() - - return {iface_name: ips} - - -def _get_public_ip(name, resource_group): - """ - Get the public ip address details by name. - """ - netconn = get_conn(client_type="network") - try: - pubip_query = netconn.public_ip_addresses.get( - resource_group_name=resource_group, public_ip_address_name=name - ) - pubip = pubip_query.as_dict() - except CloudError as exc: - salt.utils.azurearm.log_cloud_error("network", exc.message) - pubip = {"error": exc.message} - - return pubip - - -def _get_network_interface(name, resource_group): - """ - Get a network interface. - """ - public_ips = [] - private_ips = [] - netapi_versions = get_api_versions( - kwargs={ - "resource_provider": "Microsoft.Network", - "resource_type": "publicIPAddresses", - } - ) - netapi_version = netapi_versions[0] - netconn = get_conn(client_type="network") - netiface_query = netconn.network_interfaces.get( - resource_group_name=resource_group, network_interface_name=name - ) - - netiface = netiface_query.as_dict() - for index, ip_config in enumerate(netiface["ip_configurations"]): - if ip_config.get("private_ip_address") is not None: - private_ips.append(ip_config["private_ip_address"]) - if "id" in ip_config.get("public_ip_address", {}): - public_ip_name = get_resource_by_id( - ip_config["public_ip_address"]["id"], netapi_version, "name" - ) - public_ip = _get_public_ip(public_ip_name, resource_group) - public_ips.append(public_ip["ip_address"]) - netiface["ip_configurations"][index]["public_ip_address"].update(public_ip) - - return netiface, public_ips, private_ips - - -@_deprecation_message -def create_network_interface(call=None, kwargs=None): - """ - Create a network interface. - """ - if call != "action": - raise SaltCloudSystemExit( - "The create_network_interface action must be called with -a or --action." - ) - - # pylint: disable=invalid-name - IPAllocationMethod = getattr(network_models, "IPAllocationMethod") - # pylint: disable=invalid-name - NetworkInterface = getattr(network_models, "NetworkInterface") - # pylint: disable=invalid-name - NetworkInterfaceIPConfiguration = getattr( - network_models, "NetworkInterfaceIPConfiguration" - ) - # pylint: disable=invalid-name - PublicIPAddress = getattr(network_models, "PublicIPAddress") - - if not isinstance(kwargs, dict): - kwargs = {} - - vm_ = kwargs - netconn = get_conn(client_type="network") - - if kwargs.get("location") is None: - kwargs["location"] = get_location() - - if kwargs.get("network") is None: - kwargs["network"] = config.get_cloud_config_value( - "network", vm_, __opts__, search_global=False - ) - - if kwargs.get("subnet") is None: - kwargs["subnet"] = config.get_cloud_config_value( - "subnet", vm_, __opts__, search_global=False - ) - - if kwargs.get("network_resource_group") is None: - kwargs["network_resource_group"] = config.get_cloud_config_value( - "resource_group", vm_, __opts__, search_global=False - ) - - if kwargs.get("iface_name") is None: - kwargs["iface_name"] = "{}-iface0".format(vm_["name"]) - - try: - subnet_obj = netconn.subnets.get( - resource_group_name=kwargs["network_resource_group"], - virtual_network_name=kwargs["network"], - subnet_name=kwargs["subnet"], - ) - except CloudError as exc: - raise SaltCloudSystemExit( - '{} (Resource Group: "{}", VNET: "{}", Subnet: "{}")'.format( - exc.message, - kwargs["network_resource_group"], - kwargs["network"], - kwargs["subnet"], - ) - ) - - ip_kwargs = {} - ip_configurations = None - - if "load_balancer_backend_address_pools" in kwargs: - pool_dicts = kwargs["load_balancer_backend_address_pools"] - if isinstance(pool_dicts, dict): - pool_ids = [] - for load_bal, be_pools in pool_dicts.items(): - for pool in be_pools: - try: - lbbep_data = netconn.load_balancer_backend_address_pools.get( - kwargs["resource_group"], - load_bal, - pool, - ) - pool_ids.append({"id": lbbep_data.as_dict()["id"]}) - except CloudError as exc: - log.error("There was a cloud error: %s", str(exc)) - except KeyError as exc: - log.error( - "There was an error getting the Backend Pool ID: %s", - str(exc), - ) - ip_kwargs["load_balancer_backend_address_pools"] = pool_ids - - if "private_ip_address" in kwargs.keys(): - ip_kwargs["private_ip_address"] = kwargs["private_ip_address"] - ip_kwargs["private_ip_allocation_method"] = IPAllocationMethod.static - else: - ip_kwargs["private_ip_allocation_method"] = IPAllocationMethod.dynamic - - if kwargs.get("allocate_public_ip") is True: - pub_ip_name = "{}-ip".format(kwargs["iface_name"]) - poller = netconn.public_ip_addresses.create_or_update( - resource_group_name=kwargs["resource_group"], - public_ip_address_name=pub_ip_name, - parameters=PublicIPAddress( - location=kwargs["location"], - public_ip_allocation_method=IPAllocationMethod.static, - ), - ) - count = 0 - poller.wait() - while True: - try: - pub_ip_data = netconn.public_ip_addresses.get( - kwargs["resource_group"], - pub_ip_name, - ) - if pub_ip_data.ip_address: # pylint: disable=no-member - ip_kwargs["public_ip_address"] = PublicIPAddress( - id=str(pub_ip_data.id), # pylint: disable=no-member - ) - ip_configurations = [ - NetworkInterfaceIPConfiguration( - name="{}-ip".format(kwargs["iface_name"]), - subnet=subnet_obj, - **ip_kwargs - ) - ] - break - except CloudError as exc: - log.error("There was a cloud error: %s", exc) - count += 1 - if count > 120: - raise ValueError("Timed out waiting for public IP Address.") - time.sleep(5) - else: - priv_ip_name = "{}-ip".format(kwargs["iface_name"]) - ip_configurations = [ - NetworkInterfaceIPConfiguration( - name=priv_ip_name, subnet=subnet_obj, **ip_kwargs - ) - ] - - network_security_group = None - if kwargs.get("security_group") is not None: - network_security_group = netconn.network_security_groups.get( - resource_group_name=kwargs["resource_group"], - network_security_group_name=kwargs["security_group"], - ) - - iface_params = NetworkInterface( - location=kwargs["location"], - network_security_group=network_security_group, - ip_configurations=ip_configurations, - ) - - poller = netconn.network_interfaces.create_or_update( - kwargs["resource_group"], kwargs["iface_name"], iface_params - ) - try: - poller.wait() - except Exception as exc: # pylint: disable=broad-except - log.warning( - "Network interface creation could not be polled. " - "It is likely that we are reusing an existing interface. (%s)", - exc, - ) - - count = 0 - while True: - try: - return _get_network_interface( - kwargs["iface_name"], kwargs["resource_group"] - ) - except CloudError: - count += 1 - if count > 120: - raise ValueError("Timed out waiting for operation to complete.") - time.sleep(5) - - -def request_instance(vm_, kwargs=None): - """ - Request a VM from Azure. - """ - compconn = get_conn(client_type="compute") - - # pylint: disable=invalid-name - CachingTypes = getattr(compute_models, "CachingTypes") - # pylint: disable=invalid-name - DataDisk = getattr(compute_models, "DataDisk") - # pylint: disable=invalid-name - DiskCreateOptionTypes = getattr(compute_models, "DiskCreateOptionTypes") - # pylint: disable=invalid-name - HardwareProfile = getattr(compute_models, "HardwareProfile") - # pylint: disable=invalid-name - ImageReference = getattr(compute_models, "ImageReference") - # pylint: disable=invalid-name - LinuxConfiguration = getattr(compute_models, "LinuxConfiguration") - # pylint: disable=invalid-name - SshConfiguration = getattr(compute_models, "SshConfiguration") - # pylint: disable=invalid-name - SshPublicKey = getattr(compute_models, "SshPublicKey") - # pylint: disable=invalid-name - NetworkInterfaceReference = getattr(compute_models, "NetworkInterfaceReference") - # pylint: disable=invalid-name - NetworkProfile = getattr(compute_models, "NetworkProfile") - # pylint: disable=invalid-name - OSDisk = getattr(compute_models, "OSDisk") - # pylint: disable=invalid-name - OSProfile = getattr(compute_models, "OSProfile") - # pylint: disable=invalid-name - StorageProfile = getattr(compute_models, "StorageProfile") - # pylint: disable=invalid-name - VirtualHardDisk = getattr(compute_models, "VirtualHardDisk") - # pylint: disable=invalid-name - VirtualMachine = getattr(compute_models, "VirtualMachine") - # pylint: disable=invalid-name - VirtualMachineSizeTypes = getattr(compute_models, "VirtualMachineSizeTypes") - - subscription_id = config.get_cloud_config_value( - "subscription_id", get_configured_provider(), __opts__, search_global=False - ) - - if vm_.get("driver") is None: - vm_["driver"] = "azurearm" - - if vm_.get("location") is None: - vm_["location"] = get_location() - - if vm_.get("resource_group") is None: - vm_["resource_group"] = config.get_cloud_config_value( - "resource_group", vm_, __opts__, search_global=True - ) - - if vm_.get("name") is None: - vm_["name"] = config.get_cloud_config_value( - "name", vm_, __opts__, search_global=True - ) - - # pylint: disable=unused-variable - iface_data, public_ips, private_ips = create_network_interface( - call="action", kwargs=vm_ - ) - vm_["iface_id"] = iface_data["id"] - - disk_name = "{}-vol0".format(vm_["name"]) - - vm_username = config.get_cloud_config_value( - "ssh_username", - vm_, - __opts__, - search_global=True, - default=config.get_cloud_config_value( - "win_username", vm_, __opts__, search_global=True - ), - ) - - ssh_publickeyfile_contents = None - ssh_publickeyfile = config.get_cloud_config_value( - "ssh_publickeyfile", vm_, __opts__, search_global=False, default=None - ) - if ssh_publickeyfile is not None: - try: - with salt.utils.files.fopen(ssh_publickeyfile, "r") as spkc_: - ssh_publickeyfile_contents = spkc_.read() - except Exception as exc: # pylint: disable=broad-except - raise SaltCloudConfigError( - "Failed to read ssh publickey file '{}': {}".format( - ssh_publickeyfile, exc.args[-1] - ) - ) - - disable_password_authentication = config.get_cloud_config_value( - "disable_password_authentication", - vm_, - __opts__, - search_global=False, - default=False, - ) - - os_kwargs = {} - win_installer = config.get_cloud_config_value( - "win_installer", vm_, __opts__, search_global=True - ) - if not win_installer and ssh_publickeyfile_contents is not None: - sshpublickey = SshPublicKey( - key_data=ssh_publickeyfile_contents, - path="/home/{}/.ssh/authorized_keys".format(vm_username), - ) - sshconfiguration = SshConfiguration( - public_keys=[sshpublickey], - ) - linuxconfiguration = LinuxConfiguration( - disable_password_authentication=disable_password_authentication, - ssh=sshconfiguration, - ) - os_kwargs["linux_configuration"] = linuxconfiguration - vm_password = None - else: - vm_password = salt.utils.stringutils.to_str( - config.get_cloud_config_value( - "ssh_password", - vm_, - __opts__, - search_global=True, - default=config.get_cloud_config_value( - "win_password", vm_, __opts__, search_global=True - ), - ) - ) - - if win_installer or ( - vm_password is not None and not disable_password_authentication - ): - if not isinstance(vm_password, str): - raise SaltCloudSystemExit("The admin password must be a string.") - if len(vm_password) < 8 or len(vm_password) > 123: - raise SaltCloudSystemExit( - "The admin password must be between 8-123 characters long." - ) - complexity = 0 - if any(char.isdigit() for char in vm_password): - complexity += 1 - if any(char.isupper() for char in vm_password): - complexity += 1 - if any(char.islower() for char in vm_password): - complexity += 1 - if any(char in string.punctuation for char in vm_password): - complexity += 1 - if complexity < 3: - raise SaltCloudSystemExit( - "The admin password must contain at least 3 of the following types: " - "upper, lower, digits, special characters" - ) - os_kwargs["admin_password"] = vm_password - - availability_set = config.get_cloud_config_value( - "availability_set", vm_, __opts__, search_global=False, default=None - ) - if availability_set is not None and isinstance(availability_set, str): - availability_set = { - "id": "/subscriptions/{}/resourceGroups/{}/providers/Microsoft.Compute/availabilitySets/{}".format( - subscription_id, vm_["resource_group"], availability_set - ) - } - else: - availability_set = None - - cloud_env = _get_cloud_environment() - - storage_endpoint_suffix = cloud_env.suffixes.storage_endpoint - - if isinstance(vm_.get("volumes"), str): - volumes = salt.utils.yaml.safe_load(vm_["volumes"]) - else: - volumes = vm_.get("volumes") - - data_disks = None - if isinstance(volumes, list): - data_disks = [] - else: - volumes = [] - - lun = 0 - luns = [] - for volume in volumes: - if isinstance(volume, str): - volume = {"name": volume} - - volume.setdefault( - "name", - volume.get( - "name", - volume.get("name", "{}-datadisk{}".format(vm_["name"], str(lun))), - ), - ) - - volume.setdefault( - "disk_size_gb", - volume.get("logical_disk_size_in_gb", volume.get("size", 100)), - ) - # Old kwarg was host_caching, new name is caching - volume.setdefault("caching", volume.get("host_caching", "ReadOnly")) - while lun in luns: - lun += 1 - if lun > 15: - log.error("Maximum lun count has been reached") - break - volume.setdefault("lun", lun) - lun += 1 - # The default vhd is {vm_name}-datadisk{lun}.vhd - if "media_link" in volume: - volume["vhd"] = VirtualHardDisk(uri=volume["media_link"]) - del volume["media_link"] - elif volume.get("vhd") == "unmanaged": - volume["vhd"] = VirtualHardDisk( - uri="https://{}.blob.{}/vhds/{}-datadisk{}.vhd".format( - vm_["storage_account"], - storage_endpoint_suffix, - vm_["name"], - volume["lun"], - ), - ) - elif "vhd" in volume: - volume["vhd"] = VirtualHardDisk(uri=volume["vhd"]) - - if "image" in volume: - volume["create_option"] = "from_image" - elif "attach" in volume: - volume["create_option"] = "attach" - else: - volume["create_option"] = "empty" - data_disks.append(DataDisk(**volume)) - - img_ref = None - if vm_["image"].startswith("http") or vm_.get("vhd") == "unmanaged": - if vm_["image"].startswith("http"): - source_image = VirtualHardDisk(uri=vm_["image"]) - else: - source_image = None - if "|" in vm_["image"]: - img_pub, img_off, img_sku, img_ver = vm_["image"].split("|") - img_ref = ImageReference( - publisher=img_pub, - offer=img_off, - sku=img_sku, - version=img_ver, - ) - elif vm_["image"].startswith("/subscriptions"): - img_ref = ImageReference(id=vm_["image"]) - if win_installer: - os_type = "Windows" - else: - os_type = "Linux" - os_disk = OSDisk( - caching=CachingTypes.none, - create_option=DiskCreateOptionTypes.from_image, - name=disk_name, - vhd=VirtualHardDisk( - uri="https://{}.blob.{}/vhds/{}.vhd".format( - vm_["storage_account"], - storage_endpoint_suffix, - disk_name, - ), - ), - os_type=os_type, - image=source_image, - disk_size_gb=vm_.get("os_disk_size_gb"), - ) - else: - source_image = None - os_type = None - os_disk = OSDisk( - create_option=DiskCreateOptionTypes.from_image, - disk_size_gb=vm_.get("os_disk_size_gb"), - ) - if "|" in vm_["image"]: - img_pub, img_off, img_sku, img_ver = vm_["image"].split("|") - img_ref = ImageReference( - publisher=img_pub, - offer=img_off, - sku=img_sku, - version=img_ver, - ) - elif vm_["image"].startswith("/subscriptions"): - img_ref = ImageReference(id=vm_["image"]) - - userdata_file = config.get_cloud_config_value( - "userdata_file", vm_, __opts__, search_global=False, default=None - ) - userdata = config.get_cloud_config_value( - "userdata", vm_, __opts__, search_global=False, default=None - ) - userdata_template = config.get_cloud_config_value( - "userdata_template", vm_, __opts__, search_global=False, default=None - ) - - if userdata_file: - if os.path.exists(userdata_file): - with salt.utils.files.fopen(userdata_file, "r") as fh_: - userdata = fh_.read() - - if userdata and userdata_template: - userdata_sendkeys = config.get_cloud_config_value( - "userdata_sendkeys", vm_, __opts__, search_global=False, default=None - ) - if userdata_sendkeys: - vm_["priv_key"], vm_["pub_key"] = salt.utils.cloud.gen_keys( - config.get_cloud_config_value("keysize", vm_, __opts__) - ) - - key_id = vm_.get("name") - if "append_domain" in vm_: - key_id = ".".join([key_id, vm_["append_domain"]]) - - salt.utils.cloud.accept_key(__opts__["pki_dir"], vm_["pub_key"], key_id) - - userdata = salt.utils.cloud.userdata_template(__opts__, vm_, userdata) - - custom_extension = None - if userdata is not None or userdata_file is not None: - try: - if win_installer: - publisher = "Microsoft.Compute" - virtual_machine_extension_type = "CustomScriptExtension" - type_handler_version = "1.8" - if userdata_file and userdata_file.endswith(".ps1"): - command_prefix = "powershell -ExecutionPolicy Unrestricted -File " - else: - command_prefix = "" - else: - publisher = "Microsoft.Azure.Extensions" - virtual_machine_extension_type = "CustomScript" - type_handler_version = "2.0" - command_prefix = "" - - settings = {} - if userdata: - settings["commandToExecute"] = userdata - elif userdata_file.startswith("http"): - settings["fileUris"] = [userdata_file] - settings["commandToExecute"] = ( - command_prefix - + "./" - + userdata_file[userdata_file.rfind("/") + 1 :] - ) - - custom_extension = { - "resource_group": vm_["resource_group"], - "virtual_machine_name": vm_["name"], - "extension_name": vm_["name"] + "_custom_userdata_script", - "location": vm_["location"], - "publisher": publisher, - "virtual_machine_extension_type": virtual_machine_extension_type, - "type_handler_version": type_handler_version, - "auto_upgrade_minor_version": True, - "settings": settings, - "protected_settings": None, - } - except Exception as exc: # pylint: disable=broad-except - log.exception("Failed to encode userdata: %s", exc) - - params = VirtualMachine( - location=vm_["location"], - plan=None, - hardware_profile=HardwareProfile( - vm_size=getattr(VirtualMachineSizeTypes, vm_["size"].lower(), kwargs), - ), - storage_profile=StorageProfile( - os_disk=os_disk, - data_disks=data_disks, - image_reference=img_ref, - ), - os_profile=OSProfile( - admin_username=vm_username, computer_name=vm_["name"], **os_kwargs - ), - network_profile=NetworkProfile( - network_interfaces=[NetworkInterfaceReference(id=vm_["iface_id"])], - ), - availability_set=availability_set, - ) - - __utils__["cloud.fire_event"]( - "event", - "requesting instance", - "salt/cloud/{}/requesting".format(vm_["name"]), - args=__utils__["cloud.filter_event"]( - "requesting", vm_, ["name", "profile", "provider", "driver"] - ), - sock_dir=__opts__["sock_dir"], - transport=__opts__["transport"], - ) - - try: - vm_create = compconn.virtual_machines.create_or_update( - resource_group_name=vm_["resource_group"], - vm_name=vm_["name"], - parameters=params, - ) - vm_create.wait() - vm_result = vm_create.result() - vm_result = vm_result.as_dict() - if custom_extension: - create_or_update_vmextension(kwargs=custom_extension) - except CloudError as exc: - salt.utils.azurearm.log_cloud_error("compute", exc.message) - vm_result = {} - - return vm_result - - -@_deprecation_message -def create(vm_): - """ - Create a single VM from a data dict. - """ - try: - if ( - vm_["profile"] - and config.is_profile_configured( - __opts__, - _get_active_provider_name() or "azurearm", - vm_["profile"], - vm_=vm_, - ) - is False - ): - return False - except AttributeError: - pass - - if vm_.get("bootstrap_interface") is None: - vm_["bootstrap_interface"] = "public" - - __utils__["cloud.fire_event"]( - "event", - "starting create", - "salt/cloud/{}/creating".format(vm_["name"]), - args=__utils__["cloud.filter_event"]( - "creating", vm_, ["name", "profile", "provider", "driver"] - ), - sock_dir=__opts__["sock_dir"], - transport=__opts__["transport"], - ) - __utils__["cloud.cachedir_index_add"]( - vm_["name"], vm_["profile"], "azurearm", vm_["driver"] - ) - if not vm_.get("location"): - vm_["location"] = get_location(kwargs=vm_) - - log.info("Creating Cloud VM %s in %s", vm_["name"], vm_["location"]) - - vm_request = request_instance(vm_=vm_) - - if not vm_request or "error" in vm_request: - err_message = "Error creating VM {}! ({})".format(vm_["name"], str(vm_request)) - log.error(err_message) - raise SaltCloudSystemExit(err_message) - - def _query_node_data(name, bootstrap_interface): - """ - Query node data. - """ - data = show_instance(name, call="action") - if not data: - return False - ip_address = None - if bootstrap_interface == "public": - ip_address = data["public_ips"][0] - if bootstrap_interface == "private": - ip_address = data["private_ips"][0] - if ip_address is None: - return False - return ip_address - - try: - data = salt.utils.cloud.wait_for_ip( - _query_node_data, - update_args=( - vm_["name"], - vm_["bootstrap_interface"], - ), - timeout=config.get_cloud_config_value( - "wait_for_ip_timeout", vm_, __opts__, default=10 * 60 - ), - interval=config.get_cloud_config_value( - "wait_for_ip_interval", vm_, __opts__, default=10 - ), - interval_multiplier=config.get_cloud_config_value( - "wait_for_ip_interval_multiplier", vm_, __opts__, default=1 - ), - ) - except ( - SaltCloudExecutionTimeout, - SaltCloudExecutionFailure, - SaltCloudSystemExit, - ) as exc: - try: - log.warning(exc) - finally: - raise SaltCloudSystemExit(str(exc)) - - vm_["ssh_host"] = data - if not vm_.get("ssh_username"): - vm_["ssh_username"] = config.get_cloud_config_value( - "ssh_username", vm_, __opts__ - ) - vm_["password"] = config.get_cloud_config_value("ssh_password", vm_, __opts__) - ret = __utils__["cloud.bootstrap"](vm_, __opts__) - - data = show_instance(vm_["name"], call="action") - log.info("Created Cloud VM '%s'", vm_["name"]) - log.debug("'%s' VM creation details:\n%s", vm_["name"], pprint.pformat(data)) - - ret.update(data) - - __utils__["cloud.fire_event"]( - "event", - "created instance", - "salt/cloud/{}/created".format(vm_["name"]), - args=__utils__["cloud.filter_event"]( - "created", vm_, ["name", "profile", "provider", "driver"] - ), - sock_dir=__opts__["sock_dir"], - transport=__opts__["transport"], - ) - - return ret - - -@_deprecation_message -def destroy(name, call=None, kwargs=None): # pylint: disable=unused-argument - """ - Destroy a VM. - - CLI Examples: - - .. code-block:: bash - - salt-cloud -d myminion - salt-cloud -a destroy myminion service_name=myservice - """ - if kwargs is None: - kwargs = {} - - if call == "function": - raise SaltCloudSystemExit( - "The destroy action must be called with -d, --destroy, -a or --action." - ) - - compconn = get_conn(client_type="compute") - - node_data = show_instance(name, call="action") - if node_data["storage_profile"]["os_disk"].get("managed_disk"): - vhd = node_data["storage_profile"]["os_disk"]["managed_disk"]["id"] - else: - vhd = node_data["storage_profile"]["os_disk"]["vhd"]["uri"] - - ret = {name: {}} - log.debug("Deleting VM") - result = compconn.virtual_machines.delete(node_data["resource_group"], name) - result.wait() - - if __opts__.get("update_cachedir", False) is True: - __utils__["cloud.delete_minion_cachedir"]( - name, _get_active_provider_name().split(":")[0], __opts__ - ) - - cleanup_disks = config.get_cloud_config_value( - "cleanup_disks", - get_configured_provider(), - __opts__, - search_global=False, - default=False, - ) - - if cleanup_disks: - cleanup_vhds = kwargs.get( - "delete_vhd", - config.get_cloud_config_value( - "cleanup_vhds", - get_configured_provider(), - __opts__, - search_global=False, - default=False, - ), - ) - - if cleanup_vhds: - log.debug("Deleting vhd") - - comps = vhd.split("/") - container = comps[-2] - blob = comps[-1] - - ret[name]["delete_disk"] = { - "delete_disks": cleanup_disks, - "delete_vhd": cleanup_vhds, - "container": container, - "blob": blob, - } - - if vhd.startswith("http"): - ret[name]["data"] = delete_blob( - kwargs={"container": container, "blob": blob}, call="function" - ) - else: - ret[name]["data"] = delete_managed_disk( - kwargs={ - "resource_group": node_data["resource_group"], - "container": container, - "blob": blob, - }, - call="function", - ) - - cleanup_data_disks = kwargs.get( - "delete_data_disks", - config.get_cloud_config_value( - "cleanup_data_disks", - get_configured_provider(), - __opts__, - search_global=False, - default=False, - ), - ) - - if cleanup_data_disks: - log.debug("Deleting data_disks") - ret[name]["data_disks"] = {} - - for disk in node_data["storage_profile"]["data_disks"]: - datavhd = disk.get("managed_disk", {}).get("id") or disk.get( - "vhd", {} - ).get("uri") - comps = datavhd.split("/") - container = comps[-2] - blob = comps[-1] - - ret[name]["data_disks"][disk["name"]] = { - "delete_disks": cleanup_disks, - "delete_vhd": cleanup_vhds, - "container": container, - "blob": blob, - } - - if datavhd.startswith("http"): - ret[name]["data"] = delete_blob( - kwargs={"container": container, "blob": blob}, call="function" - ) - else: - ret[name]["data"] = delete_managed_disk( - kwargs={ - "resource_group": node_data["resource_group"], - "container": container, - "blob": blob, - }, - call="function", - ) - - cleanup_interfaces = config.get_cloud_config_value( - "cleanup_interfaces", - get_configured_provider(), - __opts__, - search_global=False, - default=False, - ) - - if cleanup_interfaces: - ret[name]["cleanup_network"] = { - "cleanup_interfaces": cleanup_interfaces, - "resource_group": node_data["resource_group"], - "data": [], - } - - ifaces = node_data["network_profile"]["network_interfaces"] - for iface in ifaces: - resource_group = iface["id"].split("/")[4] - ret[name]["cleanup_network"]["data"].append( - delete_interface( - kwargs={ - "resource_group": resource_group, - "iface_name": iface["name"], - }, - call="function", - ) - ) - - return ret - - -@_deprecation_message -def list_storage_accounts(call=None): - """ - List storage accounts within the subscription. - """ - if call == "action": - raise SaltCloudSystemExit( - "The list_storage_accounts function must be called with -f or --function" - ) - - storconn = get_conn(client_type="storage") - - ret = {} - try: - accounts_query = storconn.storage_accounts.list() - accounts = salt.utils.azurearm.paged_object_to_list(accounts_query) - for account in accounts: - ret[account["name"]] = account - except CloudError as exc: - salt.utils.azurearm.log_cloud_error("storage", exc.message) - ret = {"Error": exc.message} - - return ret - - -def _get_cloud_environment(): - """ - Get the cloud environment object. - """ - cloud_environment = config.get_cloud_config_value( - "cloud_environment", get_configured_provider(), __opts__, search_global=False - ) - try: - cloud_env_module = importlib.import_module("msrestazure.azure_cloud") - cloud_env = getattr(cloud_env_module, cloud_environment or "AZURE_PUBLIC_CLOUD") - except (AttributeError, ImportError): - raise SaltCloudSystemExit( - "The azure {} cloud environment is not available.".format(cloud_environment) - ) - - return cloud_env - - -def _get_block_blob_service(kwargs=None): - """ - Get the block blob storage service. - """ - resource_group = kwargs.get("resource_group") or config.get_cloud_config_value( - "resource_group", get_configured_provider(), __opts__, search_global=False - ) - sas_token = kwargs.get("sas_token") or config.get_cloud_config_value( - "sas_token", get_configured_provider(), __opts__, search_global=False - ) - storage_account = kwargs.get("storage_account") or config.get_cloud_config_value( - "storage_account", get_configured_provider(), __opts__, search_global=False - ) - storage_key = kwargs.get("storage_key") or config.get_cloud_config_value( - "storage_key", get_configured_provider(), __opts__, search_global=False - ) - - if not resource_group: - raise SaltCloudSystemExit("A resource group must be specified") - - if not storage_account: - raise SaltCloudSystemExit("A storage account must be specified") - - if not storage_key: - storconn = get_conn(client_type="storage") - storage_keys = storconn.storage_accounts.list_keys( - resource_group, storage_account - ) - storage_keys = {v.key_name: v.value for v in storage_keys.keys} - storage_key = next(iter(storage_keys.values())) - - cloud_env = _get_cloud_environment() - - endpoint_suffix = cloud_env.suffixes.storage_endpoint - - return BlockBlobService( - storage_account, - storage_key, - sas_token=sas_token, - endpoint_suffix=endpoint_suffix, - ) - - -@_deprecation_message -def list_blobs(call=None, kwargs=None): # pylint: disable=unused-argument - """ - List blobs. - """ - if kwargs is None: - kwargs = {} - - if "container" not in kwargs: - raise SaltCloudSystemExit("A container must be specified") - - storageservice = _get_block_blob_service(kwargs) - - ret = {} - try: - for blob in storageservice.list_blobs(kwargs["container"]).items: - ret[blob.name] = { - "blob_type": blob.properties.blob_type, - "last_modified": blob.properties.last_modified.isoformat(), - "server_encrypted": blob.properties.server_encrypted, - } - except Exception as exc: # pylint: disable=broad-except - log.warning(str(exc)) - - return ret - - -@_deprecation_message -def delete_blob(call=None, kwargs=None): # pylint: disable=unused-argument - """ - Delete a blob from a container. - """ - if kwargs is None: - kwargs = {} - - if "container" not in kwargs: - raise SaltCloudSystemExit("A container must be specified") - - if "blob" not in kwargs: - raise SaltCloudSystemExit("A blob must be specified") - - storageservice = _get_block_blob_service(kwargs) - - storageservice.delete_blob(kwargs["container"], kwargs["blob"]) - return True - - -@_deprecation_message -def delete_managed_disk(call=None, kwargs=None): # pylint: disable=unused-argument - """ - Delete a managed disk from a resource group. - """ - - compconn = get_conn(client_type="compute") - - try: - compconn.disks.delete(kwargs["resource_group"], kwargs["blob"]) - except Exception as exc: # pylint: disable=broad-except - log.error( - "Error deleting managed disk %s - %s", - kwargs.get("blob"), - str(exc), - ) - return False - - return True - - -@_deprecation_message -def list_virtual_networks(call=None, kwargs=None): - """ - List virtual networks. - """ - if kwargs is None: - kwargs = {} - - if call == "action": - raise SaltCloudSystemExit( - "The avail_sizes function must be called with -f or --function" - ) - - netconn = get_conn(client_type="network") - resource_groups = list_resource_groups() - - ret = {} - for group in resource_groups: - try: - networks = netconn.virtual_networks.list(resource_group_name=group) - except CloudError: - networks = {} - for network_obj in networks: - network = network_obj.as_dict() - ret[network["name"]] = network - ret[network["name"]]["subnets"] = list_subnets( - kwargs={"resource_group": group, "network": network["name"]} - ) - - return ret - - -@_deprecation_message -def list_subnets(call=None, kwargs=None): - """ - List subnets in a virtual network. - """ - if kwargs is None: - kwargs = {} - - if call == "action": - raise SaltCloudSystemExit( - "The avail_sizes function must be called with -f or --function" - ) - - netconn = get_conn(client_type="network") - - resource_group = kwargs.get("resource_group") or config.get_cloud_config_value( - "resource_group", get_configured_provider(), __opts__, search_global=False - ) - - if not resource_group and "group" in kwargs and "resource_group" not in kwargs: - resource_group = kwargs["group"] - - if not resource_group: - raise SaltCloudSystemExit("A resource group must be specified") - - if kwargs.get("network") is None: - kwargs["network"] = config.get_cloud_config_value( - "network", get_configured_provider(), __opts__, search_global=False - ) - - if "network" not in kwargs or kwargs["network"] is None: - raise SaltCloudSystemExit('A "network" must be specified') - - ret = {} - subnets = netconn.subnets.list(resource_group, kwargs["network"]) - for subnet in subnets: - ret[subnet.name] = subnet.as_dict() - ret[subnet.name]["ip_configurations"] = {} - for ip_ in subnet.ip_configurations: - comps = ip_.id.split("/") - name = comps[-1] - ret[subnet.name]["ip_configurations"][name] = ip_.as_dict() - ret[subnet.name]["ip_configurations"][name]["subnet"] = subnet.name - ret[subnet.name]["resource_group"] = resource_group - return ret - - -@_deprecation_message -def create_or_update_vmextension( - call=None, kwargs=None -): # pylint: disable=unused-argument - """ - .. versionadded:: 2019.2.0 - - Create or update a VM extension object "inside" of a VM object. - - required kwargs: - .. code-block:: yaml - - extension_name: myvmextension - virtual_machine_name: myvm - settings: {"commandToExecute": "hostname"} - - optional kwargs: - .. code-block:: yaml - - resource_group: < inferred from cloud configs > - location: < inferred from cloud configs > - publisher: < default: Microsoft.Azure.Extensions > - virtual_machine_extension_type: < default: CustomScript > - type_handler_version: < default: 2.0 > - auto_upgrade_minor_version: < default: True > - protected_settings: < default: None > - """ - if kwargs is None: - kwargs = {} - - if "extension_name" not in kwargs: - raise SaltCloudSystemExit("An extension name must be specified") - - if "virtual_machine_name" not in kwargs: - raise SaltCloudSystemExit("A virtual machine name must be specified") - - compconn = get_conn(client_type="compute") - - # pylint: disable=invalid-name - VirtualMachineExtension = getattr(compute_models, "VirtualMachineExtension") - - resource_group = kwargs.get("resource_group") or config.get_cloud_config_value( - "resource_group", get_configured_provider(), __opts__, search_global=False - ) - - if not resource_group: - raise SaltCloudSystemExit("A resource group must be specified") - - location = kwargs.get("location") or get_location() - - if not location: - raise SaltCloudSystemExit("A location must be specified") - - publisher = kwargs.get("publisher", "Microsoft.Azure.Extensions") - virtual_machine_extension_type = kwargs.get( - "virtual_machine_extension_type", "CustomScript" - ) - type_handler_version = kwargs.get("type_handler_version", "2.0") - auto_upgrade_minor_version = kwargs.get("auto_upgrade_minor_version", True) - settings = kwargs.get("settings", {}) - protected_settings = kwargs.get("protected_settings") - - if not isinstance(settings, dict): - raise SaltCloudSystemExit("VM extension settings are not valid") - elif "commandToExecute" not in settings and "script" not in settings: - raise SaltCloudSystemExit( - "VM extension settings are not valid. Either commandToExecute or script" - " must be specified." - ) - - log.info("Creating VM extension %s", kwargs["extension_name"]) - - ret = {} - try: - params = VirtualMachineExtension( - location=location, - publisher=publisher, - virtual_machine_extension_type=virtual_machine_extension_type, - type_handler_version=type_handler_version, - auto_upgrade_minor_version=auto_upgrade_minor_version, - settings=settings, - protected_settings=protected_settings, - ) - poller = compconn.virtual_machine_extensions.create_or_update( - resource_group, - kwargs["virtual_machine_name"], - kwargs["extension_name"], - params, - ) - ret = poller.result() - ret = ret.as_dict() - - except CloudError as exc: - salt.utils.azurearm.log_cloud_error( - "compute", - "Error attempting to create the VM extension: {}".format(exc.message), - ) - ret = {"error": exc.message} - - return ret - - -@_deprecation_message -def stop(name, call=None): - """ - .. versionadded:: 2019.2.0 - - Stop (deallocate) a VM - - CLI Examples: - - .. code-block:: bash - - salt-cloud -a stop myminion - """ - if call == "function": - raise SaltCloudSystemExit("The stop action must be called with -a or --action.") - - compconn = get_conn(client_type="compute") - - resource_group = config.get_cloud_config_value( - "resource_group", get_configured_provider(), __opts__, search_global=False - ) - - ret = {} - if not resource_group: - groups = list_resource_groups() - for group in groups: - try: - instance = compconn.virtual_machines.deallocate( - vm_name=name, resource_group_name=group - ) - instance.wait() - vm_result = instance.result() - ret = vm_result.as_dict() - break - except CloudError as exc: - if "was not found" in exc.message: - continue - else: - ret = {"error": exc.message} - if not ret: - salt.utils.azurearm.log_cloud_error( - "compute", "Unable to find virtual machine with name: {}".format(name) - ) - ret = {"error": "Unable to find virtual machine with name: {}".format(name)} - else: - try: - instance = compconn.virtual_machines.deallocate( - vm_name=name, resource_group_name=resource_group - ) - instance.wait() - vm_result = instance.result() - ret = vm_result.as_dict() - except CloudError as exc: - salt.utils.azurearm.log_cloud_error( - "compute", "Error attempting to stop {}: {}".format(name, exc.message) - ) - ret = {"error": exc.message} - - return ret - - -@_deprecation_message -def start(name, call=None): - """ - .. versionadded:: 2019.2.0 - - Start a VM - - CLI Examples: - - .. code-block:: bash - - salt-cloud -a start myminion - """ - if call == "function": - raise SaltCloudSystemExit( - "The start action must be called with -a or --action." - ) - - compconn = get_conn(client_type="compute") - - resource_group = config.get_cloud_config_value( - "resource_group", get_configured_provider(), __opts__, search_global=False - ) - - ret = {} - if not resource_group: - groups = list_resource_groups() - for group in groups: - try: - instance = compconn.virtual_machines.start( - vm_name=name, resource_group_name=group - ) - instance.wait() - vm_result = instance.result() - ret = vm_result.as_dict() - break - except CloudError as exc: - if "was not found" in exc.message: - continue - else: - ret = {"error": exc.message} - if not ret: - salt.utils.azurearm.log_cloud_error( - "compute", "Unable to find virtual machine with name: {}".format(name) - ) - ret = {"error": "Unable to find virtual machine with name: {}".format(name)} - else: - try: - instance = compconn.virtual_machines.start( - vm_name=name, resource_group_name=resource_group - ) - instance.wait() - vm_result = instance.result() - ret = vm_result.as_dict() - except CloudError as exc: - salt.utils.azurearm.log_cloud_error( - "compute", - "Error attempting to start {}: {}".format(name, exc.message), - ) - ret = {"error": exc.message} - - return ret diff --git a/salt/cloud/clouds/msazure.py b/salt/cloud/clouds/msazure.py deleted file mode 100644 index 8d5b74e701c..00000000000 --- a/salt/cloud/clouds/msazure.py +++ /dev/null @@ -1,3665 +0,0 @@ -""" -Azure Cloud Module -================== - -The Azure cloud module is used to control access to Microsoft Azure - -.. warning:: - - This cloud provider will be removed from Salt in version 3007 due to - the deprecation of the "Classic" API for Azure. Please migrate to - `Azure Resource Manager by March 1, 2023 - `_ - -:depends: - * `Microsoft Azure SDK for Python `_ >= 1.0.2 - * python-requests, for Python < 2.7.9 -:configuration: - Required provider parameters: - - * ``apikey`` - * ``certificate_path`` - * ``subscription_id`` - * ``backend`` - - A Management Certificate (.pem and .crt files) must be created and the .pem - file placed on the same machine that salt-cloud is run from. Information on - creating the pem file to use, and uploading the associated cer file can be - found at: - - http://www.windowsazure.com/en-us/develop/python/how-to-guides/service-management/ - - For users with Python < 2.7.9, ``backend`` must currently be set to ``requests``. - -Example ``/etc/salt/cloud.providers`` or -``/etc/salt/cloud.providers.d/azure.conf`` configuration: - -.. code-block:: yaml - - my-azure-config: - driver: azure - subscription_id: 3287abc8-f98a-c678-3bde-326766fd3617 - certificate_path: /etc/salt/azure.pem - management_host: management.core.windows.net -""" -# pylint: disable=function-redefined - -import copy -import logging -import pprint -import time -from functools import wraps - -import salt.config as config -import salt.utils.args -import salt.utils.cloud -import salt.utils.stringutils -import salt.utils.versions -import salt.utils.yaml -from salt.exceptions import SaltCloudSystemExit - -HAS_LIBS = False -try: - import azure - import azure.servicemanagement - import azure.storage - from azure.common import ( - AzureConflictHttpError, - AzureException, - AzureMissingResourceHttpError, - ) - - import salt.utils.msazure - from salt.utils.msazure import object_to_dict - - HAS_LIBS = True -except ImportError: - pass - -__virtualname__ = "azure" - - -# Get logging started -log = logging.getLogger(__name__) - - -# Only load in this module if the AZURE configurations are in place -def __virtual__(): - """ - Check for Azure configurations. - """ - if get_configured_provider() is False: - return False - - if get_dependencies() is False: - return False - - return __virtualname__ - - -def _get_active_provider_name(): - try: - return __active_provider_name__.value() - except AttributeError: - return __active_provider_name__ - - -def _deprecation_message(function): - """ - Decorator wrapper to warn about msazure deprecation - """ - - @wraps(function) - def wrapped(*args, **kwargs): - salt.utils.versions.warn_until( - "Chlorine", - "This cloud provider will be removed from Salt in version 3007 due to " - "the deprecation of the 'Classic' API for Azure. Please migrate to " - "Azure Resource Manager by March 1, 2023 " - "(https://docs.microsoft.com/en-us/azure/virtual-machines/classic-vm-deprecation)", - category=FutureWarning, - ) - ret = function(*args, **salt.utils.args.clean_kwargs(**kwargs)) - return ret - - return wrapped - - -def get_configured_provider(): - """ - Return the first configured instance. - """ - return config.is_provider_configured( - __opts__, - _get_active_provider_name() or __virtualname__, - ("subscription_id", "certificate_path"), - ) - - -@_deprecation_message -def get_dependencies(): - """ - Warn if dependencies aren't met. - """ - return config.check_driver_dependencies(__virtualname__, {"azure": HAS_LIBS}) - - -@_deprecation_message -def get_conn(): - """ - Return a conn object for the passed VM data - """ - certificate_path = config.get_cloud_config_value( - "certificate_path", get_configured_provider(), __opts__, search_global=False - ) - subscription_id = salt.utils.stringutils.to_str( - config.get_cloud_config_value( - "subscription_id", get_configured_provider(), __opts__, search_global=False - ) - ) - management_host = config.get_cloud_config_value( - "management_host", - get_configured_provider(), - __opts__, - search_global=False, - default="management.core.windows.net", - ) - return azure.servicemanagement.ServiceManagementService( - subscription_id, certificate_path, management_host - ) - - -@_deprecation_message -def script(vm_): - """ - Return the script deployment object - """ - return salt.utils.cloud.os_script( - config.get_cloud_config_value("script", vm_, __opts__), - vm_, - __opts__, - salt.utils.cloud.salt_config_to_yaml( - salt.utils.cloud.minion_config(__opts__, vm_) - ), - ) - - -@_deprecation_message -def avail_locations(conn=None, call=None): - """ - List available locations for Azure - """ - if call == "action": - raise SaltCloudSystemExit( - "The avail_locations function must be called with " - "-f or --function, or with the --list-locations option" - ) - - if not conn: - conn = get_conn() - - ret = {} - locations = conn.list_locations() - for location in locations: - ret[location.name] = { - "name": location.name, - "display_name": location.display_name, - "available_services": location.available_services, - } - return ret - - -@_deprecation_message -def avail_images(conn=None, call=None): - """ - List available images for Azure - """ - if call == "action": - raise SaltCloudSystemExit( - "The avail_images function must be called with " - "-f or --function, or with the --list-images option" - ) - - if not conn: - conn = get_conn() - - ret = {} - for item in conn.list_os_images(): - ret[item.name] = object_to_dict(item) - for item in conn.list_vm_images(): - ret[item.name] = object_to_dict(item) - return ret - - -@_deprecation_message -def avail_sizes(call=None): - """ - Return a list of sizes from Azure - """ - if call == "action": - raise SaltCloudSystemExit( - "The avail_sizes function must be called with " - "-f or --function, or with the --list-sizes option" - ) - - conn = get_conn() - data = conn.list_role_sizes() - ret = {} - for item in data.role_sizes: - ret[item.name] = object_to_dict(item) - return ret - - -@_deprecation_message -def list_nodes(conn=None, call=None): - """ - List VMs on this Azure account - """ - if call == "action": - raise SaltCloudSystemExit( - "The list_nodes function must be called with -f or --function." - ) - - ret = {} - nodes = list_nodes_full(conn, call) - for node in nodes: - ret[node] = {"name": node} - for prop in ("id", "image", "size", "state", "private_ips", "public_ips"): - ret[node][prop] = nodes[node].get(prop) - return ret - - -@_deprecation_message -def list_nodes_full(conn=None, call=None): - """ - List VMs on this Azure account, with full information - """ - if call == "action": - raise SaltCloudSystemExit( - "The list_nodes_full function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - ret = {} - services = list_hosted_services(conn=conn, call=call) - for service in services: - for deployment in services[service]["deployments"]: - deploy_dict = services[service]["deployments"][deployment] - deploy_dict_no_role_info = copy.deepcopy(deploy_dict) - del deploy_dict_no_role_info["role_list"] - del deploy_dict_no_role_info["role_instance_list"] - roles = deploy_dict["role_list"] - for role in roles: - role_instances = deploy_dict["role_instance_list"] - ret[role] = roles[role] - ret[role].update(role_instances[role]) - ret[role]["id"] = role - ret[role]["hosted_service"] = service - if role_instances[role]["power_state"] == "Started": - ret[role]["state"] = "running" - elif role_instances[role]["power_state"] == "Stopped": - ret[role]["state"] = "stopped" - else: - ret[role]["state"] = "pending" - ret[role]["private_ips"] = [] - ret[role]["public_ips"] = [] - ret[role]["deployment"] = deploy_dict_no_role_info - ret[role]["url"] = deploy_dict["url"] - ip_address = role_instances[role]["ip_address"] - if ip_address: - if salt.utils.cloud.is_public_ip(ip_address): - ret[role]["public_ips"].append(ip_address) - else: - ret[role]["private_ips"].append(ip_address) - ret[role]["size"] = role_instances[role]["instance_size"] - ret[role]["image"] = roles[role]["role_info"]["os_virtual_hard_disk"][ - "source_image_name" - ] - return ret - - -@_deprecation_message -def list_hosted_services(conn=None, call=None): - """ - List VMs on this Azure account, with full information - """ - if call == "action": - raise SaltCloudSystemExit( - "The list_hosted_services function must be called with -f or --function" - ) - - if not conn: - conn = get_conn() - - ret = {} - services = conn.list_hosted_services() - for service in services: - props = service.hosted_service_properties - ret[service.service_name] = { - "name": service.service_name, - "url": service.url, - "affinity_group": props.affinity_group, - "date_created": props.date_created, - "date_last_modified": props.date_last_modified, - "description": props.description, - "extended_properties": props.extended_properties, - "label": props.label, - "location": props.location, - "status": props.status, - "deployments": {}, - } - deployments = conn.get_hosted_service_properties( - service_name=service.service_name, embed_detail=True - ) - for deployment in deployments.deployments: - ret[service.service_name]["deployments"][deployment.name] = { - "configuration": deployment.configuration, - "created_time": deployment.created_time, - "deployment_slot": deployment.deployment_slot, - "extended_properties": deployment.extended_properties, - "input_endpoint_list": deployment.input_endpoint_list, - "label": deployment.label, - "last_modified_time": deployment.last_modified_time, - "locked": deployment.locked, - "name": deployment.name, - "persistent_vm_downtime_info": deployment.persistent_vm_downtime_info, - "private_id": deployment.private_id, - "role_instance_list": {}, - "role_list": {}, - "rollback_allowed": deployment.rollback_allowed, - "sdk_version": deployment.sdk_version, - "status": deployment.status, - "upgrade_domain_count": deployment.upgrade_domain_count, - "upgrade_status": deployment.upgrade_status, - "url": deployment.url, - } - for role_instance in deployment.role_instance_list: - ret[service.service_name]["deployments"][deployment.name][ - "role_instance_list" - ][role_instance.role_name] = { - "fqdn": role_instance.fqdn, - "instance_error_code": role_instance.instance_error_code, - "instance_fault_domain": role_instance.instance_fault_domain, - "instance_name": role_instance.instance_name, - "instance_size": role_instance.instance_size, - "instance_state_details": role_instance.instance_state_details, - "instance_status": role_instance.instance_status, - "instance_upgrade_domain": role_instance.instance_upgrade_domain, - "ip_address": role_instance.ip_address, - "power_state": role_instance.power_state, - "role_name": role_instance.role_name, - } - for role in deployment.role_list: - ret[service.service_name]["deployments"][deployment.name]["role_list"][ - role.role_name - ] = { - "role_name": role.role_name, - "os_version": role.os_version, - } - role_info = conn.get_role( - service_name=service.service_name, - deployment_name=deployment.name, - role_name=role.role_name, - ) - ret[service.service_name]["deployments"][deployment.name]["role_list"][ - role.role_name - ]["role_info"] = { - "availability_set_name": role_info.availability_set_name, - "configuration_sets": role_info.configuration_sets, - "data_virtual_hard_disks": role_info.data_virtual_hard_disks, - "os_version": role_info.os_version, - "role_name": role_info.role_name, - "role_size": role_info.role_size, - "role_type": role_info.role_type, - } - ret[service.service_name]["deployments"][deployment.name]["role_list"][ - role.role_name - ]["role_info"]["os_virtual_hard_disk"] = { - "disk_label": role_info.os_virtual_hard_disk.disk_label, - "disk_name": role_info.os_virtual_hard_disk.disk_name, - "host_caching": role_info.os_virtual_hard_disk.host_caching, - "media_link": role_info.os_virtual_hard_disk.media_link, - "os": role_info.os_virtual_hard_disk.os, - "source_image_name": role_info.os_virtual_hard_disk.source_image_name, - } - return ret - - -@_deprecation_message -def list_nodes_select(conn=None, call=None): - """ - Return a list of the VMs that are on the provider, with select fields - """ - if not conn: - conn = get_conn() - - return salt.utils.cloud.list_nodes_select( - list_nodes_full(conn, "function"), - __opts__["query.selection"], - call, - ) - - -@_deprecation_message -def show_instance(name, call=None): - """ - Show the details from the provider concerning an instance - """ - if call != "action": - raise SaltCloudSystemExit( - "The show_instance action must be called with -a or --action." - ) - - nodes = list_nodes_full() - # Find under which cloud service the name is listed, if any - if name not in nodes: - return {} - if "name" not in nodes[name]: - nodes[name]["name"] = nodes[name]["id"] - try: - __utils__["cloud.cache_node"]( - nodes[name], _get_active_provider_name(), __opts__ - ) - except TypeError: - log.warning( - "Unable to show cache node data; this may be because the node has been" - " deleted" - ) - return nodes[name] - - -@_deprecation_message -def create(vm_): - """ - Create a single VM from a data dict - """ - try: - # Check for required profile parameters before sending any API calls. - if ( - vm_["profile"] - and config.is_profile_configured( - __opts__, - _get_active_provider_name() or "azure", - vm_["profile"], - vm_=vm_, - ) - is False - ): - return False - except AttributeError: - pass - - __utils__["cloud.fire_event"]( - "event", - "starting create", - "salt/cloud/{}/creating".format(vm_["name"]), - args=__utils__["cloud.filter_event"]( - "creating", vm_, ["name", "profile", "provider", "driver"] - ), - sock_dir=__opts__["sock_dir"], - transport=__opts__["transport"], - ) - - log.info("Creating Cloud VM %s", vm_["name"]) - conn = get_conn() - - label = vm_.get("label", vm_["name"]) - service_name = vm_.get("service_name", vm_["name"]) - service_kwargs = { - "service_name": service_name, - "label": label, - "description": vm_.get("desc", vm_["name"]), - } - - loc_error = False - if "location" in vm_: - if "affinity_group" in vm_: - loc_error = True - else: - service_kwargs["location"] = vm_["location"] - elif "affinity_group" in vm_: - service_kwargs["affinity_group"] = vm_["affinity_group"] - else: - loc_error = True - - if loc_error: - raise SaltCloudSystemExit( - "Either a location or affinity group must be specified, but not both" - ) - - ssh_port = config.get_cloud_config_value( - "port", vm_, __opts__, default=22, search_global=True - ) - - ssh_endpoint = azure.servicemanagement.ConfigurationSetInputEndpoint( - name="SSH", - protocol="TCP", - port=ssh_port, - local_port=22, - ) - - network_config = azure.servicemanagement.ConfigurationSet() - network_config.input_endpoints.input_endpoints.append(ssh_endpoint) - network_config.configuration_set_type = "NetworkConfiguration" - - if "win_username" in vm_: - system_config = azure.servicemanagement.WindowsConfigurationSet( - computer_name=vm_["name"], - admin_username=vm_["win_username"], - admin_password=vm_["win_password"], - ) - - smb_port = "445" - if "smb_port" in vm_: - smb_port = vm_["smb_port"] - - smb_endpoint = azure.servicemanagement.ConfigurationSetInputEndpoint( - name="SMB", - protocol="TCP", - port=smb_port, - local_port=smb_port, - ) - - network_config.input_endpoints.input_endpoints.append(smb_endpoint) - - # Domain and WinRM configuration not yet supported by Salt Cloud - system_config.domain_join = None - system_config.win_rm = None - - else: - system_config = azure.servicemanagement.LinuxConfigurationSet( - host_name=vm_["name"], - user_name=vm_["ssh_username"], - user_password=vm_["ssh_password"], - disable_ssh_password_authentication=False, - ) - - # TODO: Might need to create a storage account - media_link = vm_["media_link"] - # TODO: Probably better to use more than just the name in the media_link - media_link += "/{}.vhd".format(vm_["name"]) - os_hd = azure.servicemanagement.OSVirtualHardDisk(vm_["image"], media_link) - - vm_kwargs = { - "service_name": service_name, - "deployment_name": service_name, - "deployment_slot": vm_["slot"], - "label": label, - "role_name": vm_["name"], - "system_config": system_config, - "os_virtual_hard_disk": os_hd, - "role_size": vm_["size"], - "network_config": network_config, - } - - if "virtual_network_name" in vm_: - vm_kwargs["virtual_network_name"] = vm_["virtual_network_name"] - if "subnet_name" in vm_: - network_config.subnet_names.append(vm_["subnet_name"]) - - log.debug("vm_kwargs: %s", vm_kwargs) - - event_kwargs = { - "service_kwargs": service_kwargs.copy(), - "vm_kwargs": vm_kwargs.copy(), - } - del event_kwargs["vm_kwargs"]["system_config"] - del event_kwargs["vm_kwargs"]["os_virtual_hard_disk"] - del event_kwargs["vm_kwargs"]["network_config"] - __utils__["cloud.fire_event"]( - "event", - "requesting instance", - "salt/cloud/{}/requesting".format(vm_["name"]), - args=__utils__["cloud.filter_event"]( - "requesting", event_kwargs, list(event_kwargs) - ), - sock_dir=__opts__["sock_dir"], - transport=__opts__["transport"], - ) - log.debug("vm_kwargs: %s", vm_kwargs) - - # Azure lets you open winrm on a new VM - # Can open up specific ports in Azure; but not on Windows - try: - conn.create_hosted_service(**service_kwargs) - except AzureConflictHttpError: - log.debug("Cloud service already exists") - except Exception as exc: # pylint: disable=broad-except - error = "The hosted service name is invalid." - if error in str(exc): - log.error( - "Error creating %s on Azure.\n\n" - "The hosted service name is invalid. The name can contain " - "only letters, numbers, and hyphens. The name must start with " - "a letter and must end with a letter or a number.", - vm_["name"], - # Show the traceback if the debug logging level is enabled - exc_info_on_loglevel=logging.DEBUG, - ) - else: - log.error( - "Error creating %s on Azure\n\n" - "The following exception was thrown when trying to " - "run the initial deployment: \n%s", - vm_["name"], - exc, - # Show the traceback if the debug logging level is enabled - exc_info_on_loglevel=logging.DEBUG, - ) - return False - try: - result = conn.create_virtual_machine_deployment(**vm_kwargs) - log.debug("Request ID for machine: %s", result.request_id) - _wait_for_async(conn, result.request_id) - except AzureConflictHttpError: - log.debug("Conflict error. The deployment may already exist, trying add_role") - # Deleting two useless keywords - del vm_kwargs["deployment_slot"] - del vm_kwargs["label"] - del vm_kwargs["virtual_network_name"] - result = conn.add_role(**vm_kwargs) # pylint: disable=unexpected-keyword-arg - _wait_for_async(conn, result.request_id) - except Exception as exc: # pylint: disable=broad-except - error = "The hosted service name is invalid." - if error in str(exc): - log.error( - "Error creating %s on Azure.\n\n" - "The VM name is invalid. The name can contain " - "only letters, numbers, and hyphens. The name must start with " - "a letter and must end with a letter or a number.", - vm_["name"], - # Show the traceback if the debug logging level is enabled - exc_info_on_loglevel=logging.DEBUG, - ) - else: - log.error( - "Error creating %s on Azure.\n\n" - "The Virtual Machine could not be created. If you " - "are using an already existing Cloud Service, " - "make sure you set up the `port` variable corresponding " - "to the SSH port exists and that the port number is not " - "already in use.\nThe following exception was thrown when trying to " - "run the initial deployment: \n%s", - vm_["name"], - exc, - # Show the traceback if the debug logging level is enabled - exc_info_on_loglevel=logging.DEBUG, - ) - return False - - def wait_for_hostname(): - """ - Wait for the IP address to become available - """ - try: - conn.get_role(service_name, service_name, vm_["name"]) - data = show_instance(vm_["name"], call="action") - if "url" in data and data["url"] != "": - return data["url"] - except AzureMissingResourceHttpError: - pass - time.sleep(1) - return False - - hostname = salt.utils.cloud.wait_for_fun( - wait_for_hostname, - timeout=config.get_cloud_config_value( - "wait_for_fun_timeout", vm_, __opts__, default=15 * 60 - ), - ) - - if not hostname: - log.error("Failed to get a value for the hostname.") - return False - - vm_["ssh_host"] = hostname.replace("http://", "").replace("/", "") - vm_["password"] = config.get_cloud_config_value("ssh_password", vm_, __opts__) - ret = __utils__["cloud.bootstrap"](vm_, __opts__) - - # Attaching volumes - volumes = config.get_cloud_config_value( - "volumes", vm_, __opts__, search_global=True - ) - if volumes: - __utils__["cloud.fire_event"]( - "event", - "attaching volumes", - "salt/cloud/{}/attaching_volumes".format(vm_["name"]), - args=__utils__["cloud.filter_event"]("attaching_volumes", vm_, ["volumes"]), - sock_dir=__opts__["sock_dir"], - transport=__opts__["transport"], - ) - - log.info("Create and attach volumes to node %s", vm_["name"]) - created = create_attach_volumes( - vm_["name"], - { - "volumes": volumes, - "service_name": service_name, - "deployment_name": vm_["name"], - "media_link": media_link, - "role_name": vm_["name"], - "del_all_vols_on_destroy": vm_.get( - "set_del_all_vols_on_destroy", False - ), - }, - call="action", - ) - ret["Attached Volumes"] = created - - data = show_instance(vm_["name"], call="action") - log.info("Created Cloud VM '%s'", vm_) - log.debug("'%s' VM creation details:\n%s", vm_["name"], pprint.pformat(data)) - - ret.update(data) - - __utils__["cloud.fire_event"]( - "event", - "created instance", - "salt/cloud/{}/created".format(vm_["name"]), - args=__utils__["cloud.filter_event"]( - "created", vm_, ["name", "profile", "provider", "driver"] - ), - sock_dir=__opts__["sock_dir"], - transport=__opts__["transport"], - ) - - return ret - - -@_deprecation_message -def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True): - """ - Create and attach volumes to created node - """ - if call != "action": - raise SaltCloudSystemExit( - "The create_attach_volumes action must be called with -a or --action." - ) - - if kwargs is None: - kwargs = {} - - if isinstance(kwargs["volumes"], str): - volumes = salt.utils.yaml.safe_load(kwargs["volumes"]) - else: - volumes = kwargs["volumes"] - - # From the Azure .NET SDK doc - # - # The Create Data Disk operation adds a data disk to a virtual - # machine. There are three ways to create the data disk using the - # Add Data Disk operation. - # Option 1 - Attach an empty data disk to - # the role by specifying the disk label and location of the disk - # image. Do not include the DiskName and SourceMediaLink elements in - # the request body. Include the MediaLink element and reference a - # blob that is in the same geographical region as the role. You can - # also omit the MediaLink element. In this usage, Azure will create - # the data disk in the storage account configured as default for the - # role. - # Option 2 - Attach an existing data disk that is in the image - # repository. Do not include the DiskName and SourceMediaLink - # elements in the request body. Specify the data disk to use by - # including the DiskName element. Note: If included the in the - # response body, the MediaLink and LogicalDiskSizeInGB elements are - # ignored. - # Option 3 - Specify the location of a blob in your storage - # account that contain a disk image to use. Include the - # SourceMediaLink element. Note: If the MediaLink element - # isincluded, it is ignored. (see - # http://msdn.microsoft.com/en-us/library/windowsazure/jj157199.aspx - # for more information) - # - # Here only option 1 is implemented - conn = get_conn() - ret = [] - for volume in volumes: - if "disk_name" in volume: - log.error("You cannot specify a disk_name. Only new volumes are allowed") - return False - # Use the size keyword to set a size, but you can use the - # azure name too. If neither is set, the disk has size 100GB - volume.setdefault("logical_disk_size_in_gb", volume.get("size", 100)) - volume.setdefault("host_caching", "ReadOnly") - volume.setdefault("lun", 0) - # The media link is vm_name-disk-[0-15].vhd - volume.setdefault( - "media_link", - kwargs["media_link"][:-4] + "-disk-{}.vhd".format(volume["lun"]), - ) - volume.setdefault( - "disk_label", kwargs["role_name"] + "-disk-{}".format(volume["lun"]) - ) - volume_dict = {"volume_name": volume["lun"], "disk_label": volume["disk_label"]} - - # Preparing the volume dict to be passed with ** - kwargs_add_data_disk = [ - "lun", - "host_caching", - "media_link", - "disk_label", - "disk_name", - "logical_disk_size_in_gb", - "source_media_link", - ] - for key in set(volume.keys()) - set(kwargs_add_data_disk): - del volume[key] - - attach = conn.add_data_disk( - kwargs["service_name"], - kwargs["deployment_name"], - kwargs["role_name"], - **volume - ) - log.debug(attach) - - # If attach is None then everything is fine - if attach: - msg = "{} attached to {} (aka {})".format( - volume_dict["volume_name"], - kwargs["role_name"], - name, - ) - log.info(msg) - ret.append(msg) - else: - log.error("Error attaching %s on Azure", volume_dict) - return ret - - -@_deprecation_message -def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True): - """ - Create and attach volumes to created node - """ - if call != "action": - raise SaltCloudSystemExit( - "The create_attach_volumes action must be called with -a or --action." - ) - - if kwargs is None: - kwargs = {} - - if isinstance(kwargs["volumes"], str): - volumes = salt.utils.yaml.safe_load(kwargs["volumes"]) - else: - volumes = kwargs["volumes"] - - # From the Azure .NET SDK doc - # - # The Create Data Disk operation adds a data disk to a virtual - # machine. There are three ways to create the data disk using the - # Add Data Disk operation. - # Option 1 - Attach an empty data disk to - # the role by specifying the disk label and location of the disk - # image. Do not include the DiskName and SourceMediaLink elements in - # the request body. Include the MediaLink element and reference a - # blob that is in the same geographical region as the role. You can - # also omit the MediaLink element. In this usage, Azure will create - # the data disk in the storage account configured as default for the - # role. - # Option 2 - Attach an existing data disk that is in the image - # repository. Do not include the DiskName and SourceMediaLink - # elements in the request body. Specify the data disk to use by - # including the DiskName element. Note: If included the in the - # response body, the MediaLink and LogicalDiskSizeInGB elements are - # ignored. - # Option 3 - Specify the location of a blob in your storage - # account that contain a disk image to use. Include the - # SourceMediaLink element. Note: If the MediaLink element - # isincluded, it is ignored. (see - # http://msdn.microsoft.com/en-us/library/windowsazure/jj157199.aspx - # for more information) - # - # Here only option 1 is implemented - conn = get_conn() - ret = [] - for volume in volumes: - if "disk_name" in volume: - log.error("You cannot specify a disk_name. Only new volumes are allowed") - return False - # Use the size keyword to set a size, but you can use the - # azure name too. If neither is set, the disk has size 100GB - volume.setdefault("logical_disk_size_in_gb", volume.get("size", 100)) - volume.setdefault("host_caching", "ReadOnly") - volume.setdefault("lun", 0) - # The media link is vm_name-disk-[0-15].vhd - volume.setdefault( - "media_link", - kwargs["media_link"][:-4] + "-disk-{}.vhd".format(volume["lun"]), - ) - volume.setdefault( - "disk_label", kwargs["role_name"] + "-disk-{}".format(volume["lun"]) - ) - volume_dict = {"volume_name": volume["lun"], "disk_label": volume["disk_label"]} - - # Preparing the volume dict to be passed with ** - kwargs_add_data_disk = [ - "lun", - "host_caching", - "media_link", - "disk_label", - "disk_name", - "logical_disk_size_in_gb", - "source_media_link", - ] - for key in set(volume.keys()) - set(kwargs_add_data_disk): - del volume[key] - - result = conn.add_data_disk( - kwargs["service_name"], - kwargs["deployment_name"], - kwargs["role_name"], - **volume - ) - _wait_for_async(conn, result.request_id) - - msg = "{} attached to {} (aka {})".format( - volume_dict["volume_name"], kwargs["role_name"], name - ) - log.info(msg) - ret.append(msg) - return ret - - -# Helper function for azure tests -def _wait_for_async(conn, request_id): - """ - Helper function for azure tests - """ - count = 0 - log.debug("Waiting for asynchronous operation to complete") - result = conn.get_operation_status(request_id) - while result.status == "InProgress": - count = count + 1 - if count > 120: - raise ValueError( - "Timed out waiting for asynchronous operation to complete." - ) - time.sleep(5) - result = conn.get_operation_status(request_id) - - if result.status != "Succeeded": - raise AzureException( - "Operation failed. {message} ({code})".format( - message=result.error.message, code=result.error.code - ) - ) - - -@_deprecation_message -def destroy(name, conn=None, call=None, kwargs=None): - """ - Destroy a VM - - CLI Examples: - - .. code-block:: bash - - salt-cloud -d myminion - salt-cloud -a destroy myminion service_name=myservice - """ - if call == "function": - raise SaltCloudSystemExit( - "The destroy action must be called with -d, --destroy, -a or --action." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - instance_data = show_instance(name, call="action") - service_name = instance_data["deployment"]["name"] - disk_name = instance_data["role_info"]["os_virtual_hard_disk"]["disk_name"] - - ret = {} - # TODO: Add the ability to delete or not delete a hosted service when - # deleting a VM - try: - log.debug("Deleting role") - result = conn.delete_role(service_name, service_name, name) - delete_type = "delete_role" - except AzureException: - log.debug("Failed to delete role, deleting deployment") - try: - result = conn.delete_deployment(service_name, service_name) - except AzureConflictHttpError as exc: - log.error(exc.message) - raise SaltCloudSystemExit("{}: {}".format(name, exc.message)) - delete_type = "delete_deployment" - _wait_for_async(conn, result.request_id) - ret[name] = { - delete_type: {"request_id": result.request_id}, - } - if __opts__.get("update_cachedir", False) is True: - __utils__["cloud.delete_minion_cachedir"]( - name, _get_active_provider_name().split(":")[0], __opts__ - ) - - cleanup_disks = config.get_cloud_config_value( - "cleanup_disks", - get_configured_provider(), - __opts__, - search_global=False, - default=False, - ) - if cleanup_disks: - cleanup_vhds = kwargs.get( - "delete_vhd", - config.get_cloud_config_value( - "cleanup_vhds", - get_configured_provider(), - __opts__, - search_global=False, - default=False, - ), - ) - log.debug("Deleting disk %s", disk_name) - if cleanup_vhds: - log.debug("Deleting vhd") - - def wait_for_destroy(): - """ - Wait for the VM to be deleted - """ - try: - data = delete_disk( - kwargs={"name": disk_name, "delete_vhd": cleanup_vhds}, - call="function", - ) - return data - except AzureConflictHttpError: - log.debug("Waiting for VM to be destroyed...") - time.sleep(5) - return False - - data = salt.utils.cloud.wait_for_fun( - wait_for_destroy, - timeout=config.get_cloud_config_value( - "wait_for_fun_timeout", {}, __opts__, default=15 * 60 - ), - ) - ret[name]["delete_disk"] = { - "name": disk_name, - "delete_vhd": cleanup_vhds, - "data": data, - } - - # Services can't be cleaned up unless disks are too - cleanup_services = config.get_cloud_config_value( - "cleanup_services", - get_configured_provider(), - __opts__, - search_global=False, - default=False, - ) - if cleanup_services: - log.debug("Deleting service %s", service_name) - - def wait_for_disk_delete(): - """ - Wait for the disk to be deleted - """ - try: - data = delete_service( - kwargs={"name": service_name}, call="function" - ) - return data - except AzureConflictHttpError: - log.debug("Waiting for disk to be deleted...") - time.sleep(5) - return False - - data = salt.utils.cloud.wait_for_fun( - wait_for_disk_delete, - timeout=config.get_cloud_config_value( - "wait_for_fun_timeout", {}, __opts__, default=15 * 60 - ), - ) - ret[name]["delete_services"] = {"name": service_name, "data": data} - - return ret - - -@_deprecation_message -def list_storage_services(conn=None, call=None): - """ - List VMs on this Azure account, with full information - """ - if call != "function": - raise SaltCloudSystemExit( - "The list_storage_services function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - ret = {} - accounts = conn.list_storage_accounts() - for service in accounts.storage_services: - ret[service.service_name] = { - "capabilities": service.capabilities, - "service_name": service.service_name, - "storage_service_properties": service.storage_service_properties, - "extended_properties": service.extended_properties, - "storage_service_keys": service.storage_service_keys, - "url": service.url, - } - return ret - - -@_deprecation_message -def get_operation_status(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Get Operation Status, based on a request ID - - CLI Example: - - .. code-block:: bash - - salt-cloud -f get_operation_status my-azure id=0123456789abcdef0123456789abcdef - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_instance function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "id" not in kwargs: - raise SaltCloudSystemExit('A request ID must be specified as "id"') - - if not conn: - conn = get_conn() - - data = conn.get_operation_status(kwargs["id"]) - ret = { - "http_status_code": data.http_status_code, - "id": kwargs["id"], - "status": data.status, - } - if hasattr(data.error, "code"): - ret["error"] = { - "code": data.error.code, - "message": data.error.message, - } - - return ret - - -@_deprecation_message -def list_storage(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - List storage accounts associated with the account - - CLI Example: - - .. code-block:: bash - - salt-cloud -f list_storage my-azure - """ - if call != "function": - raise SaltCloudSystemExit( - "The list_storage function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - data = conn.list_storage_accounts() - pprint.pprint(dir(data)) - ret = {} - for item in data.storage_services: - ret[item.service_name] = object_to_dict(item) - return ret - - -@_deprecation_message -def show_storage(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - List storage service properties - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_storage my-azure name=my_storage - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_storage function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - data = conn.get_storage_account_properties( - kwargs["name"], - ) - return object_to_dict(data) - - -# To reflect the Azure API -get_storage = show_storage - - -@_deprecation_message -def show_storage_keys(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Show storage account keys - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_storage_keys my-azure name=my_storage - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_storage_keys function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - try: - data = conn.get_storage_account_keys( - kwargs["name"], - ) - except AzureMissingResourceHttpError as exc: - storage_data = show_storage(kwargs={"name": kwargs["name"]}, call="function") - if storage_data["storage_service_properties"]["status"] == "Creating": - raise SaltCloudSystemExit( - "The storage account keys have not yet been created." - ) - else: - raise SaltCloudSystemExit("{}: {}".format(kwargs["name"], exc.message)) - return object_to_dict(data) - - -# To reflect the Azure API -get_storage_keys = show_storage_keys - - -@_deprecation_message -def create_storage(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Create a new storage account - - CLI Example: - - .. code-block:: bash - - salt-cloud -f create_storage my-azure name=my_storage label=my_storage location='West US' - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_storage function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if not conn: - conn = get_conn() - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - if "description" not in kwargs: - raise SaltCloudSystemExit('A description must be specified as "description"') - - if "label" not in kwargs: - raise SaltCloudSystemExit('A label must be specified as "label"') - - if "location" not in kwargs and "affinity_group" not in kwargs: - raise SaltCloudSystemExit( - "Either a location or an affinity_group must be specified (but not both)" - ) - - try: - data = conn.create_storage_account( - service_name=kwargs["name"], - label=kwargs["label"], - description=kwargs.get("description", None), - location=kwargs.get("location", None), - affinity_group=kwargs.get("affinity_group", None), - extended_properties=kwargs.get("extended_properties", None), - geo_replication_enabled=kwargs.get("geo_replication_enabled", None), - account_type=kwargs.get("account_type", "Standard_GRS"), - ) - return {"Success": "The storage account was successfully created"} - except AzureConflictHttpError: - raise SaltCloudSystemExit( - "There was a conflict. This usually means that the storage account already" - " exists." - ) - - -@_deprecation_message -def update_storage(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Update a storage account's properties - - CLI Example: - - .. code-block:: bash - - salt-cloud -f update_storage my-azure name=my_storage label=my_storage - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_storage function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - data = conn.update_storage_account( - service_name=kwargs["name"], - label=kwargs.get("label", None), - description=kwargs.get("description", None), - extended_properties=kwargs.get("extended_properties", None), - geo_replication_enabled=kwargs.get("geo_replication_enabled", None), - account_type=kwargs.get("account_type", "Standard_GRS"), - ) - return show_storage(kwargs={"name": kwargs["name"]}, call="function") - - -@_deprecation_message -def regenerate_storage_keys(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Regenerate storage account keys. Requires a key_type ("primary" or - "secondary") to be specified. - - CLI Example: - - .. code-block:: bash - - salt-cloud -f regenerate_storage_keys my-azure name=my_storage key_type=primary - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_storage function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - if "key_type" not in kwargs or kwargs["key_type"] not in ("primary", "secondary"): - raise SaltCloudSystemExit( - 'A key_type must be specified ("primary" or "secondary")' - ) - - try: - data = conn.regenerate_storage_account_keys( - service_name=kwargs["name"], - key_type=kwargs["key_type"], - ) - return show_storage_keys(kwargs={"name": kwargs["name"]}, call="function") - except AzureConflictHttpError: - raise SaltCloudSystemExit( - "There was a conflict. This usually means that the storage account already" - " exists." - ) - - -@_deprecation_message -def delete_storage(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Delete a specific storage account - - CLI Examples: - - .. code-block:: bash - - salt-cloud -f delete_storage my-azure name=my_storage - """ - if call != "function": - raise SaltCloudSystemExit( - "The delete_storage function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - if not conn: - conn = get_conn() - - try: - data = conn.delete_storage_account(kwargs["name"]) - return {"Success": "The storage account was successfully deleted"} - except AzureMissingResourceHttpError as exc: - raise SaltCloudSystemExit("{}: {}".format(kwargs["name"], exc.message)) - - -@_deprecation_message -def list_services(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - List hosted services associated with the account - - CLI Example: - - .. code-block:: bash - - salt-cloud -f list_services my-azure - """ - if call != "function": - raise SaltCloudSystemExit( - "The list_services function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - data = conn.list_hosted_services() - ret = {} - for item in data.hosted_services: - ret[item.service_name] = object_to_dict(item) - ret[item.service_name]["name"] = item.service_name - return ret - - -@_deprecation_message -def show_service(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - List hosted service properties - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_service my-azure name=my_service - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_service function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - data = conn.get_hosted_service_properties( - kwargs["name"], kwargs.get("details", False) - ) - ret = object_to_dict(data) - return ret - - -@_deprecation_message -def create_service(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Create a new hosted service - - CLI Example: - - .. code-block:: bash - - salt-cloud -f create_service my-azure name=my_service label=my_service location='West US' - """ - if call != "function": - raise SaltCloudSystemExit( - "The create_service function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - if "label" not in kwargs: - raise SaltCloudSystemExit('A label must be specified as "label"') - - if "location" not in kwargs and "affinity_group" not in kwargs: - raise SaltCloudSystemExit( - "Either a location or an affinity_group must be specified (but not both)" - ) - - try: - data = conn.create_hosted_service( - kwargs["name"], - kwargs["label"], - kwargs.get("description", None), - kwargs.get("location", None), - kwargs.get("affinity_group", None), - kwargs.get("extended_properties", None), - ) - return {"Success": "The service was successfully created"} - except AzureConflictHttpError: - raise SaltCloudSystemExit( - "There was a conflict. This usually means that the service already exists." - ) - - -@_deprecation_message -def delete_service(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Delete a specific service associated with the account - - CLI Examples: - - .. code-block:: bash - - salt-cloud -f delete_service my-azure name=my_service - """ - if call != "function": - raise SaltCloudSystemExit( - "The delete_service function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - if not conn: - conn = get_conn() - - try: - conn.delete_hosted_service(kwargs["name"]) - return {"Success": "The service was successfully deleted"} - except AzureMissingResourceHttpError as exc: - raise SaltCloudSystemExit("{}: {}".format(kwargs["name"], exc.message)) - - -@_deprecation_message -def list_disks(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - List disks associated with the account - - CLI Example: - - .. code-block:: bash - - salt-cloud -f list_disks my-azure - """ - if call != "function": - raise SaltCloudSystemExit( - "The list_disks function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - data = conn.list_disks() - ret = {} - for item in data.disks: - ret[item.name] = object_to_dict(item) - return ret - - -@_deprecation_message -def show_disk(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Return information about a disk - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_disk my-azure name=my_disk - """ - if call != "function": - raise SaltCloudSystemExit( - "The get_disk function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - data = conn.get_disk(kwargs["name"]) - return object_to_dict(data) - - -# For consistency with Azure SDK -get_disk = show_disk - - -@_deprecation_message -def cleanup_unattached_disks(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Cleans up all disks associated with the account, which are not attached. - *** CAUTION *** This is a destructive function with no undo button, and no - "Are you sure?" confirmation! - - CLI Examples: - - .. code-block:: bash - - salt-cloud -f cleanup_unattached_disks my-azure name=my_disk - salt-cloud -f cleanup_unattached_disks my-azure name=my_disk delete_vhd=True - """ - if call != "function": - raise SaltCloudSystemExit( - "The delete_disk function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - disks = list_disks(kwargs=kwargs, conn=conn, call="function") - for disk in disks: - if disks[disk]["attached_to"] is None: - del_kwargs = { - "name": disks[disk]["name"], - "delete_vhd": kwargs.get("delete_vhd", False), - } - log.info( - "Deleting disk %s, deleting VHD: %s", - del_kwargs["name"], - del_kwargs["delete_vhd"], - ) - data = delete_disk(kwargs=del_kwargs, call="function") - return True - - -@_deprecation_message -def delete_disk(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Delete a specific disk associated with the account - - CLI Examples: - - .. code-block:: bash - - salt-cloud -f delete_disk my-azure name=my_disk - salt-cloud -f delete_disk my-azure name=my_disk delete_vhd=True - """ - if call != "function": - raise SaltCloudSystemExit( - "The delete_disk function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - if not conn: - conn = get_conn() - - try: - data = conn.delete_disk(kwargs["name"], kwargs.get("delete_vhd", False)) - return {"Success": "The disk was successfully deleted"} - except AzureMissingResourceHttpError as exc: - raise SaltCloudSystemExit("{}: {}".format(kwargs["name"], exc.message)) - - -@_deprecation_message -def update_disk(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Update a disk's properties - - CLI Example: - - .. code-block:: bash - - salt-cloud -f update_disk my-azure name=my_disk label=my_disk - salt-cloud -f update_disk my-azure name=my_disk new_name=another_disk - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_disk function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - old_data = show_disk(kwargs={"name": kwargs["name"]}, call="function") - data = conn.update_disk( - disk_name=kwargs["name"], - has_operating_system=kwargs.get( - "has_operating_system", old_data["has_operating_system"] - ), - label=kwargs.get("label", old_data["label"]), - media_link=kwargs.get("media_link", old_data["media_link"]), - name=kwargs.get("new_name", old_data["name"]), - os=kwargs.get("os", old_data["os"]), - ) - return show_disk(kwargs={"name": kwargs["name"]}, call="function") - - -@_deprecation_message -def list_service_certificates(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - List certificates associated with the service - - CLI Example: - - .. code-block:: bash - - salt-cloud -f list_service_certificates my-azure name=my_service - """ - if call != "function": - raise SaltCloudSystemExit( - "The list_service_certificates function must be called with -f or" - " --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A service name must be specified as "name"') - - if not conn: - conn = get_conn() - - data = conn.list_service_certificates(service_name=kwargs["name"]) - ret = {} - for item in data.certificates: - ret[item.thumbprint] = object_to_dict(item) - return ret - - -@_deprecation_message -def show_service_certificate(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Return information about a service certificate - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_service_certificate my-azure name=my_service_certificate \\ - thumbalgorithm=sha1 thumbprint=0123456789ABCDEF - """ - if call != "function": - raise SaltCloudSystemExit( - "The get_service_certificate function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A service name must be specified as "name"') - - if "thumbalgorithm" not in kwargs: - raise SaltCloudSystemExit( - 'A thumbalgorithm must be specified as "thumbalgorithm"' - ) - - if "thumbprint" not in kwargs: - raise SaltCloudSystemExit('A thumbprint must be specified as "thumbprint"') - - data = conn.get_service_certificate( - kwargs["name"], - kwargs["thumbalgorithm"], - kwargs["thumbprint"], - ) - return object_to_dict(data) - - -# For consistency with Azure SDK -get_service_certificate = show_service_certificate - - -@_deprecation_message -def add_service_certificate(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Add a new service certificate - - CLI Example: - - .. code-block:: bash - - salt-cloud -f add_service_certificate my-azure name=my_service_certificate \\ - data='...CERT_DATA...' certificate_format=sha1 password=verybadpass - """ - if call != "function": - raise SaltCloudSystemExit( - "The add_service_certificate function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - if "data" not in kwargs: - raise SaltCloudSystemExit('Certificate data must be specified as "data"') - - if "certificate_format" not in kwargs: - raise SaltCloudSystemExit( - 'A certificate_format must be specified as "certificate_format"' - ) - - if "password" not in kwargs: - raise SaltCloudSystemExit('A password must be specified as "password"') - - try: - data = conn.add_service_certificate( - kwargs["name"], - kwargs["data"], - kwargs["certificate_format"], - kwargs["password"], - ) - return {"Success": "The service certificate was successfully added"} - except AzureConflictHttpError: - raise SaltCloudSystemExit( - "There was a conflict. This usually means that the " - "service certificate already exists." - ) - - -@_deprecation_message -def delete_service_certificate(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Delete a specific certificate associated with the service - - CLI Examples: - - .. code-block:: bash - - salt-cloud -f delete_service_certificate my-azure name=my_service_certificate \\ - thumbalgorithm=sha1 thumbprint=0123456789ABCDEF - """ - if call != "function": - raise SaltCloudSystemExit( - "The delete_service_certificate function must be called with -f or" - " --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - if "thumbalgorithm" not in kwargs: - raise SaltCloudSystemExit( - 'A thumbalgorithm must be specified as "thumbalgorithm"' - ) - - if "thumbprint" not in kwargs: - raise SaltCloudSystemExit('A thumbprint must be specified as "thumbprint"') - - if not conn: - conn = get_conn() - - try: - data = conn.delete_service_certificate( - kwargs["name"], - kwargs["thumbalgorithm"], - kwargs["thumbprint"], - ) - return {"Success": "The service certificate was successfully deleted"} - except AzureMissingResourceHttpError as exc: - raise SaltCloudSystemExit("{}: {}".format(kwargs["name"], exc.message)) - - -@_deprecation_message -def list_management_certificates(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - List management certificates associated with the subscription - - CLI Example: - - .. code-block:: bash - - salt-cloud -f list_management_certificates my-azure name=my_management - """ - if call != "function": - raise SaltCloudSystemExit( - "The list_management_certificates function must be called with -f or" - " --function." - ) - - if not conn: - conn = get_conn() - - data = conn.list_management_certificates() - ret = {} - for item in data.subscription_certificates: - ret[item.subscription_certificate_thumbprint] = object_to_dict(item) - return ret - - -@_deprecation_message -def show_management_certificate(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Return information about a management_certificate - - CLI Example: - - .. code-block:: bash - - salt-cloud -f get_management_certificate my-azure name=my_management_certificate \\ - thumbalgorithm=sha1 thumbprint=0123456789ABCDEF - """ - if call != "function": - raise SaltCloudSystemExit( - "The get_management_certificate function must be called with -f or" - " --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "thumbprint" not in kwargs: - raise SaltCloudSystemExit('A thumbprint must be specified as "thumbprint"') - - data = conn.get_management_certificate(kwargs["thumbprint"]) - return object_to_dict(data) - - -# For consistency with Azure SDK -get_management_certificate = show_management_certificate - - -@_deprecation_message -def add_management_certificate(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Add a new management certificate - - CLI Example: - - .. code-block:: bash - - salt-cloud -f add_management_certificate my-azure public_key='...PUBKEY...' \\ - thumbprint=0123456789ABCDEF data='...CERT_DATA...' - """ - if call != "function": - raise SaltCloudSystemExit( - "The add_management_certificate function must be called with -f or" - " --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "public_key" not in kwargs: - raise SaltCloudSystemExit('A public_key must be specified as "public_key"') - - if "thumbprint" not in kwargs: - raise SaltCloudSystemExit('A thumbprint must be specified as "thumbprint"') - - if "data" not in kwargs: - raise SaltCloudSystemExit('Certificate data must be specified as "data"') - - try: - conn.add_management_certificate( - kwargs["name"], - kwargs["thumbprint"], - kwargs["data"], - ) - return {"Success": "The management certificate was successfully added"} - except AzureConflictHttpError: - raise SaltCloudSystemExit( - "There was a conflict. " - "This usually means that the management certificate already exists." - ) - - -@_deprecation_message -def delete_management_certificate(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Delete a specific certificate associated with the management - - CLI Examples: - - .. code-block:: bash - - salt-cloud -f delete_management_certificate my-azure name=my_management_certificate \\ - thumbalgorithm=sha1 thumbprint=0123456789ABCDEF - """ - if call != "function": - raise SaltCloudSystemExit( - "The delete_management_certificate function must be called with -f or" - " --function." - ) - - if kwargs is None: - kwargs = {} - - if "thumbprint" not in kwargs: - raise SaltCloudSystemExit('A thumbprint must be specified as "thumbprint"') - - if not conn: - conn = get_conn() - - try: - conn.delete_management_certificate(kwargs["thumbprint"]) - return {"Success": "The management certificate was successfully deleted"} - except AzureMissingResourceHttpError as exc: - raise SaltCloudSystemExit("{}: {}".format(kwargs["thumbprint"], exc.message)) - - -@_deprecation_message -def list_virtual_networks(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - List input endpoints associated with the deployment - - CLI Example: - - .. code-block:: bash - - salt-cloud -f list_virtual_networks my-azure service=myservice deployment=mydeployment - """ - if call != "function": - raise SaltCloudSystemExit( - "The list_virtual_networks function must be called with -f or --function." - ) - - path = "services/networking/virtualnetwork" - data = query(path) - return data - - -@_deprecation_message -def list_input_endpoints(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - List input endpoints associated with the deployment - - CLI Example: - - .. code-block:: bash - - salt-cloud -f list_input_endpoints my-azure service=myservice deployment=mydeployment - """ - if call != "function": - raise SaltCloudSystemExit( - "The list_input_endpoints function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "service" not in kwargs: - raise SaltCloudSystemExit('A service name must be specified as "service"') - - if "deployment" not in kwargs: - raise SaltCloudSystemExit('A deployment name must be specified as "deployment"') - - path = "services/hostedservices/{}/deployments/{}".format( - kwargs["service"], - kwargs["deployment"], - ) - - data = query(path) - if data is None: - raise SaltCloudSystemExit( - "There was an error listing endpoints with the {} service on the {}" - " deployment.".format(kwargs["service"], kwargs["deployment"]) - ) - - ret = {} - for item in data: - if "Role" in item: - role = item["Role"] - if not isinstance(role, dict): - return ret - input_endpoint = ( - role["ConfigurationSets"]["ConfigurationSet"] - .get("InputEndpoints", {}) - .get("InputEndpoint") - ) - if not input_endpoint: - continue - if not isinstance(input_endpoint, list): - input_endpoint = [input_endpoint] - for endpoint in input_endpoint: - ret[endpoint["Name"]] = endpoint - return ret - return ret - - -@_deprecation_message -def show_input_endpoint(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Show an input endpoint associated with the deployment - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_input_endpoint my-azure service=myservice \\ - deployment=mydeployment name=SSH - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_input_endpoint function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('An endpoint name must be specified as "name"') - - data = list_input_endpoints(kwargs=kwargs, call="function") - return data.get(kwargs["name"], None) - - -# For consistency with Azure SDK -get_input_endpoint = show_input_endpoint - - -@_deprecation_message -def update_input_endpoint(kwargs=None, conn=None, call=None, activity="update"): - """ - .. versionadded:: 2015.8.0 - - Update an input endpoint associated with the deployment. Please note that - there may be a delay before the changes show up. - - CLI Example: - - .. code-block:: bash - - salt-cloud -f update_input_endpoint my-azure service=myservice \\ - deployment=mydeployment role=myrole name=HTTP local_port=80 \\ - port=80 protocol=tcp enable_direct_server_return=False \\ - timeout_for_tcp_idle_connection=4 - """ - if call != "function": - raise SaltCloudSystemExit( - "The update_input_endpoint function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "service" not in kwargs: - raise SaltCloudSystemExit('A service name must be specified as "service"') - - if "deployment" not in kwargs: - raise SaltCloudSystemExit('A deployment name must be specified as "deployment"') - - if "name" not in kwargs: - raise SaltCloudSystemExit('An endpoint name must be specified as "name"') - - if "role" not in kwargs: - raise SaltCloudSystemExit('An role name must be specified as "role"') - - if activity != "delete": - if "port" not in kwargs: - raise SaltCloudSystemExit('An endpoint port must be specified as "port"') - - if "protocol" not in kwargs: - raise SaltCloudSystemExit( - 'An endpoint protocol (tcp or udp) must be specified as "protocol"' - ) - - if "local_port" not in kwargs: - kwargs["local_port"] = kwargs["port"] - - if "enable_direct_server_return" not in kwargs: - kwargs["enable_direct_server_return"] = False - kwargs["enable_direct_server_return"] = str( - kwargs["enable_direct_server_return"] - ).lower() - - if "timeout_for_tcp_idle_connection" not in kwargs: - kwargs["timeout_for_tcp_idle_connection"] = 4 - - old_endpoints = list_input_endpoints(kwargs, call="function") - - endpoints_xml = "" - endpoint_xml = """ - - {local_port} - {name} - {port} - {protocol} - {enable_direct_server_return} - {timeout_for_tcp_idle_connection} - """ - - if activity == "add": - old_endpoints[kwargs["name"]] = kwargs - old_endpoints[kwargs["name"]]["Name"] = kwargs["name"] - - for endpoint in old_endpoints: - if old_endpoints[endpoint]["Name"] == kwargs["name"]: - if activity != "delete": - this_endpoint_xml = endpoint_xml.format(**kwargs) - endpoints_xml += this_endpoint_xml - else: - this_endpoint_xml = endpoint_xml.format( - local_port=old_endpoints[endpoint]["LocalPort"], - name=old_endpoints[endpoint]["Name"], - port=old_endpoints[endpoint]["Port"], - protocol=old_endpoints[endpoint]["Protocol"], - enable_direct_server_return=old_endpoints[endpoint][ - "EnableDirectServerReturn" - ], - timeout_for_tcp_idle_connection=old_endpoints[endpoint].get( - "IdleTimeoutInMinutes", 4 - ), - ) - endpoints_xml += this_endpoint_xml - - request_xml = """ - - - NetworkConfiguration - {} - - - - - -""".format( - endpoints_xml - ) - - path = "services/hostedservices/{}/deployments/{}/roles/{}".format( - kwargs["service"], - kwargs["deployment"], - kwargs["role"], - ) - query( - path=path, - method="PUT", - header_dict={"Content-Type": "application/xml"}, - data=request_xml, - decode=False, - ) - return True - - -@_deprecation_message -def add_input_endpoint(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Add an input endpoint to the deployment. Please note that - there may be a delay before the changes show up. - - CLI Example: - - .. code-block:: bash - - salt-cloud -f add_input_endpoint my-azure service=myservice \\ - deployment=mydeployment role=myrole name=HTTP local_port=80 \\ - port=80 protocol=tcp enable_direct_server_return=False \\ - timeout_for_tcp_idle_connection=4 - """ - return update_input_endpoint( - kwargs=kwargs, - conn=conn, - call="function", - activity="add", - ) - - -@_deprecation_message -def delete_input_endpoint(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Delete an input endpoint from the deployment. Please note that - there may be a delay before the changes show up. - - CLI Example: - - .. code-block:: bash - - salt-cloud -f delete_input_endpoint my-azure service=myservice \\ - deployment=mydeployment role=myrole name=HTTP - """ - return update_input_endpoint( - kwargs=kwargs, - conn=conn, - call="function", - activity="delete", - ) - - -@_deprecation_message -def show_deployment(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Return information about a deployment - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_deployment my-azure name=my_deployment - """ - if call != "function": - raise SaltCloudSystemExit( - "The get_deployment function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "service_name" not in kwargs: - raise SaltCloudSystemExit('A service name must be specified as "service_name"') - - if "deployment_name" not in kwargs: - raise SaltCloudSystemExit( - 'A deployment name must be specified as "deployment_name"' - ) - - data = conn.get_deployment_by_name( - service_name=kwargs["service_name"], - deployment_name=kwargs["deployment_name"], - ) - return object_to_dict(data) - - -# For consistency with Azure SDK -get_deployment = show_deployment - - -@_deprecation_message -def list_affinity_groups(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - List input endpoints associated with the deployment - - CLI Example: - - .. code-block:: bash - - salt-cloud -f list_affinity_groups my-azure - """ - if call != "function": - raise SaltCloudSystemExit( - "The list_affinity_groups function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - data = conn.list_affinity_groups() - ret = {} - for item in data.affinity_groups: - ret[item.name] = object_to_dict(item) - return ret - - -@_deprecation_message -def show_affinity_group(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Show an affinity group associated with the account - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_affinity_group my-azure service=myservice \\ - deployment=mydeployment name=SSH - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_affinity_group function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('An affinity group name must be specified as "name"') - - data = conn.get_affinity_group_properties(affinity_group_name=kwargs["name"]) - return object_to_dict(data) - - -# For consistency with Azure SDK -get_affinity_group = show_affinity_group - - -@_deprecation_message -def create_affinity_group(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Create a new affinity group - - CLI Example: - - .. code-block:: bash - - salt-cloud -f create_affinity_group my-azure name=my_affinity_group - """ - if call != "function": - raise SaltCloudSystemExit( - "The create_affinity_group function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - if "label" not in kwargs: - raise SaltCloudSystemExit('A label must be specified as "label"') - - if "location" not in kwargs: - raise SaltCloudSystemExit('A location must be specified as "location"') - - try: - conn.create_affinity_group( - kwargs["name"], - kwargs["label"], - kwargs["location"], - kwargs.get("description", None), - ) - return {"Success": "The affinity group was successfully created"} - except AzureConflictHttpError: - raise SaltCloudSystemExit( - "There was a conflict. This usually means that the affinity group already" - " exists." - ) - - -@_deprecation_message -def update_affinity_group(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Update an affinity group's properties - - CLI Example: - - .. code-block:: bash - - salt-cloud -f update_affinity_group my-azure name=my_group label=my_group - """ - if call != "function": - raise SaltCloudSystemExit( - "The update_affinity_group function must be called with -f or --function." - ) - - if not conn: - conn = get_conn() - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - if "label" not in kwargs: - raise SaltCloudSystemExit('A label must be specified as "label"') - - conn.update_affinity_group( - affinity_group_name=kwargs["name"], - label=kwargs["label"], - description=kwargs.get("description", None), - ) - return show_affinity_group(kwargs={"name": kwargs["name"]}, call="function") - - -@_deprecation_message -def delete_affinity_group(kwargs=None, conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Delete a specific affinity group associated with the account - - CLI Examples: - - .. code-block:: bash - - salt-cloud -f delete_affinity_group my-azure name=my_affinity_group - """ - if call != "function": - raise SaltCloudSystemExit( - "The delete_affinity_group function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit('A name must be specified as "name"') - - if not conn: - conn = get_conn() - - try: - conn.delete_affinity_group(kwargs["name"]) - return {"Success": "The affinity group was successfully deleted"} - except AzureMissingResourceHttpError as exc: - raise SaltCloudSystemExit("{}: {}".format(kwargs["name"], exc.message)) - - -@_deprecation_message -def get_storage_conn(storage_account=None, storage_key=None, conn_kwargs=None): - """ - .. versionadded:: 2015.8.0 - - Return a storage_conn object for the storage account - """ - if conn_kwargs is None: - conn_kwargs = {} - - if not storage_account: - storage_account = config.get_cloud_config_value( - "storage_account", - get_configured_provider(), - __opts__, - search_global=False, - default=conn_kwargs.get("storage_account", None), - ) - if not storage_key: - storage_key = config.get_cloud_config_value( - "storage_key", - get_configured_provider(), - __opts__, - search_global=False, - default=conn_kwargs.get("storage_key", None), - ) - return azure.storage.BlobService(storage_account, storage_key) - - -@_deprecation_message -def make_blob_url(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Creates the URL to access a blob - - CLI Example: - - .. code-block:: bash - - salt-cloud -f make_blob_url my-azure container=mycontainer blob=myblob - - container: - Name of the container. - blob: - Name of the blob. - account: - Name of the storage account. If not specified, derives the host base - from the provider configuration. - protocol: - Protocol to use: 'http' or 'https'. If not specified, derives the host - base from the provider configuration. - host_base: - Live host base URL. If not specified, derives the host base from the - provider configuration. - """ - if call != "function": - raise SaltCloudSystemExit( - "The make_blob_url function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "container" not in kwargs: - raise SaltCloudSystemExit('A container name must be specified as "container"') - - if "blob" not in kwargs: - raise SaltCloudSystemExit('A blob name must be specified as "blob"') - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - data = storage_conn.make_blob_url( - kwargs["container"], - kwargs["blob"], - kwargs.get("account", None), - kwargs.get("protocol", None), - kwargs.get("host_base", None), - ) - ret = {} - for item in data.containers: - ret[item.name] = object_to_dict(item) - return ret - - -@_deprecation_message -def list_storage_containers(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - List containers associated with the storage account - - CLI Example: - - .. code-block:: bash - - salt-cloud -f list_storage_containers my-azure - """ - if call != "function": - raise SaltCloudSystemExit( - "The list_storage_containers function must be called with -f or --function." - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - data = storage_conn.list_containers() - ret = {} - for item in data.containers: - ret[item.name] = object_to_dict(item) - return ret - - -@_deprecation_message -def create_storage_container(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Create a storage container - - CLI Example: - - .. code-block:: bash - - salt-cloud -f create_storage_container my-azure name=mycontainer - - name: - Name of container to create. - meta_name_values: - Optional. A dict with name_value pairs to associate with the - container as metadata. Example:{'Category':'test'} - blob_public_access: - Optional. Possible values include: container, blob - fail_on_exist: - Specify whether to throw an exception when the container exists. - """ - if call != "function": - raise SaltCloudSystemExit( - "The create_storage_container function must be called with -f or" - " --function." - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - try: - storage_conn.create_container( - container_name=kwargs["name"], - x_ms_meta_name_values=kwargs.get("meta_name_values", None), - x_ms_blob_public_access=kwargs.get("blob_public_access", None), - fail_on_exist=kwargs.get("fail_on_exist", False), - ) - return {"Success": "The storage container was successfully created"} - except AzureConflictHttpError: - raise SaltCloudSystemExit( - "There was a conflict. This usually means that the storage container" - " already exists." - ) - - -@_deprecation_message -def show_storage_container(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Show a container associated with the storage account - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_storage_container my-azure name=myservice - - name: - Name of container to show. - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_storage_container function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit( - 'An storage container name must be specified as "name"' - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - data = storage_conn.get_container_properties( - container_name=kwargs["name"], - x_ms_lease_id=kwargs.get("lease_id", None), - ) - return data - - -# For consistency with Azure SDK -get_storage_container = show_storage_container - - -@_deprecation_message -def show_storage_container_metadata(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Show a storage container's metadata - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_storage_container_metadata my-azure name=myservice - - name: - Name of container to show. - lease_id: - If specified, show_storage_container_metadata only succeeds if the - container's lease is active and matches this ID. - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_storage_container function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit( - 'An storage container name must be specified as "name"' - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - data = storage_conn.get_container_metadata( - container_name=kwargs["name"], - x_ms_lease_id=kwargs.get("lease_id", None), - ) - return data - - -# For consistency with Azure SDK -get_storage_container_metadata = show_storage_container_metadata - - -@_deprecation_message -def set_storage_container_metadata(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Set a storage container's metadata - - CLI Example: - - .. code-block:: bash - - salt-cloud -f set_storage_container my-azure name=mycontainer \\ - x_ms_meta_name_values='{"my_name": "my_value"}' - - name: - Name of existing container. - meta_name_values: - A dict containing name, value for metadata. - Example: {'category':'test'} - lease_id: - If specified, set_storage_container_metadata only succeeds if the - container's lease is active and matches this ID. - """ - if call != "function": - raise SaltCloudSystemExit( - "The create_storage_container function must be called with -f or" - " --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit( - 'An storage container name must be specified as "name"' - ) - - x_ms_meta_name_values = salt.utils.yaml.safe_load( - kwargs.get("meta_name_values", "") - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - try: - storage_conn.set_container_metadata( - container_name=kwargs["name"], - x_ms_meta_name_values=x_ms_meta_name_values, - x_ms_lease_id=kwargs.get("lease_id", None), - ) - return {"Success": "The storage container was successfully updated"} - except AzureConflictHttpError: - raise SaltCloudSystemExit("There was a conflict.") - - -@_deprecation_message -def show_storage_container_acl(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Show a storage container's acl - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_storage_container_acl my-azure name=myservice - - name: - Name of existing container. - lease_id: - If specified, show_storage_container_acl only succeeds if the - container's lease is active and matches this ID. - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_storage_container function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit( - 'An storage container name must be specified as "name"' - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - data = storage_conn.get_container_acl( - container_name=kwargs["name"], - x_ms_lease_id=kwargs.get("lease_id", None), - ) - return data - - -# For consistency with Azure SDK -get_storage_container_acl = show_storage_container_acl - - -@_deprecation_message -def set_storage_container_acl(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Set a storage container's acl - - CLI Example: - - .. code-block:: bash - - salt-cloud -f set_storage_container my-azure name=mycontainer - - name: - Name of existing container. - signed_identifiers: - SignedIdentifers instance - blob_public_access: - Optional. Possible values include: container, blob - lease_id: - If specified, set_storage_container_acl only succeeds if the - container's lease is active and matches this ID. - """ - if call != "function": - raise SaltCloudSystemExit( - "The create_storage_container function must be called with -f or" - " --function." - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - try: - data = storage_conn.set_container_acl( - container_name=kwargs["name"], - signed_identifiers=kwargs.get("signed_identifiers", None), - x_ms_blob_public_access=kwargs.get("blob_public_access", None), - x_ms_lease_id=kwargs.get("lease_id", None), - ) - return {"Success": "The storage container was successfully updated"} - except AzureConflictHttpError: - raise SaltCloudSystemExit("There was a conflict.") - - -@_deprecation_message -def delete_storage_container(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Delete a container associated with the storage account - - CLI Example: - - .. code-block:: bash - - salt-cloud -f delete_storage_container my-azure name=mycontainer - - name: - Name of container to create. - fail_not_exist: - Specify whether to throw an exception when the container exists. - lease_id: - If specified, delete_storage_container only succeeds if the - container's lease is active and matches this ID. - """ - if call != "function": - raise SaltCloudSystemExit( - "The delete_storage_container function must be called with -f or" - " --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit( - 'An storage container name must be specified as "name"' - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - data = storage_conn.delete_container( - container_name=kwargs["name"], - fail_not_exist=kwargs.get("fail_not_exist", None), - x_ms_lease_id=kwargs.get("lease_id", None), - ) - return data - - -@_deprecation_message -def lease_storage_container(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Lease a container associated with the storage account - - CLI Example: - - .. code-block:: bash - - salt-cloud -f lease_storage_container my-azure name=mycontainer - - name: - Name of container to create. - lease_action: - Required. Possible values: acquire|renew|release|break|change - lease_id: - Required if the container has an active lease. - lease_duration: - Specifies the duration of the lease, in seconds, or negative one - (-1) for a lease that never expires. A non-infinite lease can be - between 15 and 60 seconds. A lease duration cannot be changed - using renew or change. For backwards compatibility, the default is - 60, and the value is only used on an acquire operation. - lease_break_period: - Optional. For a break operation, this is the proposed duration of - seconds that the lease should continue before it is broken, between - 0 and 60 seconds. This break period is only used if it is shorter - than the time remaining on the lease. If longer, the time remaining - on the lease is used. A new lease will not be available before the - break period has expired, but the lease may be held for longer than - the break period. If this header does not appear with a break - operation, a fixed-duration lease breaks after the remaining lease - period elapses, and an infinite lease breaks immediately. - proposed_lease_id: - Optional for acquire, required for change. Proposed lease ID, in a - GUID string format. - """ - if call != "function": - raise SaltCloudSystemExit( - "The lease_storage_container function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "name" not in kwargs: - raise SaltCloudSystemExit( - 'An storage container name must be specified as "name"' - ) - - lease_actions = ("acquire", "renew", "release", "break", "change") - - if kwargs.get("lease_action", None) not in lease_actions: - raise SaltCloudSystemExit( - "A lease_action must be one of: {}".format(", ".join(lease_actions)) - ) - - if kwargs["lease_action"] != "acquire" and "lease_id" not in kwargs: - raise SaltCloudSystemExit( - 'A lease ID must be specified for the "{}" lease action ' - 'as "lease_id"'.format(kwargs["lease_action"]) - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - data = storage_conn.lease_container( - container_name=kwargs["name"], - x_ms_lease_action=kwargs["lease_action"], - x_ms_lease_id=kwargs.get("lease_id", None), - x_ms_lease_duration=kwargs.get("lease_duration", 60), - x_ms_lease_break_period=kwargs.get("lease_break_period", None), - x_ms_proposed_lease_id=kwargs.get("proposed_lease_id", None), - ) - - return data - - -@_deprecation_message -def list_blobs(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - List blobs associated with the container - - CLI Example: - - .. code-block:: bash - - salt-cloud -f list_blobs my-azure container=mycontainer - - container: - The name of the storage container - prefix: - Optional. Filters the results to return only blobs whose names - begin with the specified prefix. - marker: - Optional. A string value that identifies the portion of the list - to be returned with the next list operation. The operation returns - a marker value within the response body if the list returned was - not complete. The marker value may then be used in a subsequent - call to request the next set of list items. The marker value is - opaque to the client. - maxresults: - Optional. Specifies the maximum number of blobs to return, - including all BlobPrefix elements. If the request does not specify - maxresults or specifies a value greater than 5,000, the server will - return up to 5,000 items. Setting maxresults to a value less than - or equal to zero results in error response code 400 (Bad Request). - include: - Optional. Specifies one or more datasets to include in the - response. To specify more than one of these options on the URI, - you must separate each option with a comma. Valid values are: - - snapshots: - Specifies that snapshots should be included in the - enumeration. Snapshots are listed from oldest to newest in - the response. - metadata: - Specifies that blob metadata be returned in the response. - uncommittedblobs: - Specifies that blobs for which blocks have been uploaded, - but which have not been committed using Put Block List - (REST API), be included in the response. - copy: - Version 2012-02-12 and newer. Specifies that metadata - related to any current or previous Copy Blob operation - should be included in the response. - delimiter: - Optional. When the request includes this parameter, the operation - returns a BlobPrefix element in the response body that acts as a - placeholder for all blobs whose names begin with the same - substring up to the appearance of the delimiter character. The - delimiter may be a single character or a string. - """ - if call != "function": - raise SaltCloudSystemExit( - "The list_blobs function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "container" not in kwargs: - raise SaltCloudSystemExit( - 'An storage container name must be specified as "container"' - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - return salt.utils.msazure.list_blobs(storage_conn=storage_conn, **kwargs) - - -@_deprecation_message -def show_blob_service_properties(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Show a blob's service properties - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_blob_service_properties my-azure - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_blob_service_properties function must be called with -f or" - " --function." - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - data = storage_conn.get_blob_service_properties( - timeout=kwargs.get("timeout", None), - ) - return data - - -# For consistency with Azure SDK -get_blob_service_properties = show_blob_service_properties - - -@_deprecation_message -def set_blob_service_properties(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Sets the properties of a storage account's Blob service, including - Windows Azure Storage Analytics. You can also use this operation to - set the default request version for all incoming requests that do not - have a version specified. - - CLI Example: - - .. code-block:: bash - - salt-cloud -f set_blob_service_properties my-azure - - properties: - a StorageServiceProperties object. - timeout: - Optional. The timeout parameter is expressed in seconds. - """ - if call != "function": - raise SaltCloudSystemExit( - "The set_blob_service_properties function must be called with -f or" - " --function." - ) - - if kwargs is None: - kwargs = {} - - if "properties" not in kwargs: - raise SaltCloudSystemExit( - 'The blob service properties name must be specified as "properties"' - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - data = storage_conn.get_blob_service_properties( - storage_service_properties=kwargs["properties"], - timeout=kwargs.get("timeout", None), - ) - return data - - -@_deprecation_message -def show_blob_properties(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Returns all user-defined metadata, standard HTTP properties, and - system properties for the blob. - - CLI Example: - - .. code-block:: bash - - salt-cloud -f show_blob_properties my-azure container=mycontainer blob=myblob - - container: - Name of existing container. - blob: - Name of existing blob. - lease_id: - Required if the blob has an active lease. - """ - if call != "function": - raise SaltCloudSystemExit( - "The show_blob_properties function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "container" not in kwargs: - raise SaltCloudSystemExit('The container name must be specified as "container"') - - if "blob" not in kwargs: - raise SaltCloudSystemExit('The blob name must be specified as "blob"') - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - try: - data = storage_conn.get_blob_properties( - container_name=kwargs["container"], - blob_name=kwargs["blob"], - x_ms_lease_id=kwargs.get("lease_id", None), - ) - except AzureMissingResourceHttpError: - raise SaltCloudSystemExit("The specified blob does not exist.") - - return data - - -# For consistency with Azure SDK -get_blob_properties = show_blob_properties - - -@_deprecation_message -def set_blob_properties(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Set a blob's properties - - CLI Example: - - .. code-block:: bash - - salt-cloud -f set_blob_properties my-azure - - container: - Name of existing container. - blob: - Name of existing blob. - blob_cache_control: - Optional. Modifies the cache control string for the blob. - blob_content_type: - Optional. Sets the blob's content type. - blob_content_md5: - Optional. Sets the blob's MD5 hash. - blob_content_encoding: - Optional. Sets the blob's content encoding. - blob_content_language: - Optional. Sets the blob's content language. - lease_id: - Required if the blob has an active lease. - blob_content_disposition: - Optional. Sets the blob's Content-Disposition header. - The Content-Disposition response header field conveys additional - information about how to process the response payload, and also can - be used to attach additional metadata. For example, if set to - attachment, it indicates that the user-agent should not display the - response, but instead show a Save As dialog with a filename other - than the blob name specified. - """ - if call != "function": - raise SaltCloudSystemExit( - "The set_blob_properties function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "container" not in kwargs: - raise SaltCloudSystemExit( - 'The blob container name must be specified as "container"' - ) - - if "blob" not in kwargs: - raise SaltCloudSystemExit('The blob name must be specified as "blob"') - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - data = storage_conn.get_blob_properties( - container_name=kwargs["container"], - blob_name=kwargs["blob"], - x_ms_blob_cache_control=kwargs.get("blob_cache_control", None), - x_ms_blob_content_type=kwargs.get("blob_content_type", None), - x_ms_blob_content_md5=kwargs.get("blob_content_md5", None), - x_ms_blob_content_encoding=kwargs.get("blob_content_encoding", None), - x_ms_blob_content_language=kwargs.get("blob_content_language", None), - x_ms_lease_id=kwargs.get("lease_id", None), - x_ms_blob_content_disposition=kwargs.get("blob_content_disposition", None), - ) - - return data - - -@_deprecation_message -def put_blob(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Upload a blob - - CLI Examples: - - .. code-block:: bash - - salt-cloud -f put_blob my-azure container=base name=top.sls blob_path=/srv/salt/top.sls - salt-cloud -f put_blob my-azure container=base name=content.txt blob_content='Some content' - - container: - Name of existing container. - name: - Name of existing blob. - blob_path: - The path on the local machine of the file to upload as a blob. Either - this or blob_content must be specified. - blob_content: - The actual content to be uploaded as a blob. Either this or blob_path - must me specified. - cache_control: - Optional. The Blob service stores this value but does not use or - modify it. - content_language: - Optional. Specifies the natural languages used by this resource. - content_md5: - Optional. An MD5 hash of the blob content. This hash is used to - verify the integrity of the blob during transport. When this header - is specified, the storage service checks the hash that has arrived - with the one that was sent. If the two hashes do not match, the - operation will fail with error code 400 (Bad Request). - blob_content_type: - Optional. Set the blob's content type. - blob_content_encoding: - Optional. Set the blob's content encoding. - blob_content_language: - Optional. Set the blob's content language. - blob_content_md5: - Optional. Set the blob's MD5 hash. - blob_cache_control: - Optional. Sets the blob's cache control. - meta_name_values: - A dict containing name, value for metadata. - lease_id: - Required if the blob has an active lease. - """ - if call != "function": - raise SaltCloudSystemExit( - "The put_blob function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "container" not in kwargs: - raise SaltCloudSystemExit( - 'The blob container name must be specified as "container"' - ) - - if "name" not in kwargs: - raise SaltCloudSystemExit('The blob name must be specified as "name"') - - if "blob_path" not in kwargs and "blob_content" not in kwargs: - raise SaltCloudSystemExit( - 'Either a path to a file needs to be passed in as "blob_path" or ' - 'the contents of a blob as "blob_content."' - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - return salt.utils.msazure.put_blob(storage_conn=storage_conn, **kwargs) - - -@_deprecation_message -def get_blob(kwargs=None, storage_conn=None, call=None): - """ - .. versionadded:: 2015.8.0 - - Download a blob - - CLI Example: - - .. code-block:: bash - - salt-cloud -f get_blob my-azure container=base name=top.sls local_path=/srv/salt/top.sls - salt-cloud -f get_blob my-azure container=base name=content.txt return_content=True - - container: - Name of existing container. - name: - Name of existing blob. - local_path: - The path on the local machine to download the blob to. Either this or - return_content must be specified. - return_content: - Whether or not to return the content directly from the blob. If - specified, must be True or False. Either this or the local_path must - be specified. - snapshot: - Optional. The snapshot parameter is an opaque DateTime value that, - when present, specifies the blob snapshot to retrieve. - lease_id: - Required if the blob has an active lease. - progress_callback: - callback for progress with signature function(current, total) where - current is the number of bytes transferred so far, and total is the - size of the blob. - max_connections: - Maximum number of parallel connections to use when the blob size - exceeds 64MB. - Set to 1 to download the blob chunks sequentially. - Set to 2 or more to download the blob chunks in parallel. This uses - more system resources but will download faster. - max_retries: - Number of times to retry download of blob chunk if an error occurs. - retry_wait: - Sleep time in secs between retries. - """ - if call != "function": - raise SaltCloudSystemExit( - "The get_blob function must be called with -f or --function." - ) - - if kwargs is None: - kwargs = {} - - if "container" not in kwargs: - raise SaltCloudSystemExit( - 'The blob container name must be specified as "container"' - ) - - if "name" not in kwargs: - raise SaltCloudSystemExit('The blob name must be specified as "name"') - - if "local_path" not in kwargs and "return_content" not in kwargs: - raise SaltCloudSystemExit( - 'Either a local path needs to be passed in as "local_path" or ' - '"return_content" to return the blob contents directly' - ) - - if not storage_conn: - storage_conn = get_storage_conn(conn_kwargs=kwargs) - - return salt.utils.msazure.get_blob(storage_conn=storage_conn, **kwargs) - - -@_deprecation_message -def query(path, method="GET", data=None, params=None, header_dict=None, decode=True): - """ - Perform a query directly against the Azure REST API - """ - certificate_path = config.get_cloud_config_value( - "certificate_path", get_configured_provider(), __opts__, search_global=False - ) - subscription_id = salt.utils.stringutils.to_str( - config.get_cloud_config_value( - "subscription_id", get_configured_provider(), __opts__, search_global=False - ) - ) - management_host = config.get_cloud_config_value( - "management_host", - get_configured_provider(), - __opts__, - search_global=False, - default="management.core.windows.net", - ) - backend = config.get_cloud_config_value( - "backend", get_configured_provider(), __opts__, search_global=False - ) - url = "https://{management_host}/{subscription_id}/{path}".format( - management_host=management_host, - subscription_id=subscription_id, - path=path, - ) - - if header_dict is None: - header_dict = {} - - header_dict["x-ms-version"] = "2014-06-01" - - result = salt.utils.http.query( - url, - method=method, - params=params, - data=data, - header_dict=header_dict, - port=443, - text=True, - cert=certificate_path, - backend=backend, - decode=decode, - decode_type="xml", - ) - if "dict" in result: - return result["dict"] - return diff --git a/salt/fileserver/azurefs.py b/salt/fileserver/azurefs.py deleted file mode 100644 index cdb1a390d7a..00000000000 --- a/salt/fileserver/azurefs.py +++ /dev/null @@ -1,396 +0,0 @@ -""" -The backend for serving files from the Azure blob storage service. - -.. versionadded:: 2015.8.0 - -To enable, add ``azurefs`` to the :conf_master:`fileserver_backend` option in -the Master config file. - -.. code-block:: yaml - - fileserver_backend: - - azurefs - -Starting in Salt 2018.3.0, this fileserver requires the standalone Azure -Storage SDK for Python. Theoretically any version >= v0.20.0 should work, but -it was developed against the v0.33.0 version. - -Each storage container will be mapped to an environment. By default, containers -will be mapped to the ``base`` environment. You can override this behavior with -the ``saltenv`` configuration option. You can have an unlimited number of -storage containers, and can have a storage container serve multiple -environments, or have multiple storage containers mapped to the same -environment. Normal first-found rules apply, and storage containers are -searched in the order they are defined. - -You must have either an account_key or a sas_token defined for each container, -if it is private. If you use a sas_token, it must have READ and LIST -permissions. - -.. code-block:: yaml - - azurefs: - - account_name: my_storage - account_key: 'fNH9cRp0+qVIVYZ+5rnZAhHc9ycOUcJnHtzpfOr0W0sxrtL2KVLuMe1xDfLwmfed+JJInZaEdWVCPHD4d/oqeA==' - container_name: my_container - - account_name: my_storage - sas_token: 'ss=b&sp=&sv=2015-07-08&sig=cohxXabx8FQdXsSEHyUXMjsSfNH2tZ2OB97Ou44pkRE%3D&srt=co&se=2017-04-18T21%3A38%3A01Z' - container_name: my_dev_container - saltenv: dev - - account_name: my_storage - container_name: my_public_container - -.. note:: - - Do not include the leading ? for sas_token if generated from the web -""" - - -import base64 -import logging -import os -import shutil - -import salt.fileserver -import salt.utils.files -import salt.utils.gzip_util -import salt.utils.hashutils -import salt.utils.json -import salt.utils.path -import salt.utils.stringutils -from salt.utils.versions import Version - -try: - import azure.storage - - if Version(azure.storage.__version__) < Version("0.20.0"): - raise ImportError("azure.storage.__version__ must be >= 0.20.0") - HAS_AZURE = True -except (ImportError, AttributeError): - HAS_AZURE = False - - -__virtualname__ = "azurefs" - -log = logging.getLogger() - - -def __virtual__(): - """ - Only load if defined in fileserver_backend and azure.storage is present - """ - if __virtualname__ not in __opts__["fileserver_backend"]: - return False - - if not HAS_AZURE: - return False - - if "azurefs" not in __opts__: - return False - - if not _validate_config(): - return False - - return True - - -def find_file(path, saltenv="base", **kwargs): - """ - Search the environment for the relative path - """ - fnd = {"path": "", "rel": ""} - for container in __opts__.get("azurefs", []): - if container.get("saltenv", "base") != saltenv: - continue - full = os.path.join(_get_container_path(container), path) - if os.path.isfile(full) and not salt.fileserver.is_file_ignored(__opts__, path): - fnd["path"] = full - fnd["rel"] = path - try: - # Converting the stat result to a list, the elements of the - # list correspond to the following stat_result params: - # 0 => st_mode=33188 - # 1 => st_ino=10227377 - # 2 => st_dev=65026 - # 3 => st_nlink=1 - # 4 => st_uid=1000 - # 5 => st_gid=1000 - # 6 => st_size=1056233 - # 7 => st_atime=1468284229 - # 8 => st_mtime=1456338235 - # 9 => st_ctime=1456338235 - fnd["stat"] = list(os.stat(full)) - except Exception: # pylint: disable=broad-except - pass - return fnd - return fnd - - -def envs(): - """ - Each container configuration can have an environment setting, or defaults - to base - """ - saltenvs = [] - for container in __opts__.get("azurefs", []): - saltenvs.append(container.get("saltenv", "base")) - # Remove duplicates - return list(set(saltenvs)) - - -def serve_file(load, fnd): - """ - Return a chunk from a file based on the data received - """ - ret = {"data": "", "dest": ""} - required_load_keys = ("path", "loc", "saltenv") - if not all(x in load for x in required_load_keys): - log.debug( - "Not all of the required keys present in payload. Missing: %s", - ", ".join(required_load_keys.difference(load)), - ) - return ret - if not fnd["path"]: - return ret - ret["dest"] = fnd["rel"] - gzip = load.get("gzip", None) - fpath = os.path.normpath(fnd["path"]) - with salt.utils.files.fopen(fpath, "rb") as fp_: - fp_.seek(load["loc"]) - data = fp_.read(__opts__["file_buffer_size"]) - if data and not salt.utils.files.is_binary(fpath): - data = data.decode(__salt_system_encoding__) - if gzip and data: - data = salt.utils.gzip_util.compress(data, gzip) - ret["gzip"] = gzip - ret["data"] = data - return ret - - -def update(): - """ - Update caches of the storage containers. - - Compares the md5 of the files on disk to the md5 of the blobs in the - container, and only updates if necessary. - - Also processes deletions by walking the container caches and comparing - with the list of blobs in the container - """ - for container in __opts__["azurefs"]: - path = _get_container_path(container) - try: - if not os.path.exists(path): - os.makedirs(path) - elif not os.path.isdir(path): - shutil.rmtree(path) - os.makedirs(path) - except Exception as exc: # pylint: disable=broad-except - log.exception("Error occurred creating cache directory for azurefs") - continue - blob_service = _get_container_service(container) - name = container["container_name"] - try: - blob_list = blob_service.list_blobs(name) - except Exception as exc: # pylint: disable=broad-except - log.exception("Error occurred fetching blob list for azurefs") - continue - - # Walk the cache directory searching for deletions - blob_names = [blob.name for blob in blob_list] - blob_set = set(blob_names) - for root, dirs, files in salt.utils.path.os_walk(path): - for f in files: - fname = os.path.join(root, f) - relpath = os.path.relpath(fname, path) - if relpath not in blob_set: - salt.fileserver.wait_lock(fname + ".lk", fname) - try: - os.unlink(fname) - except Exception: # pylint: disable=broad-except - pass - if not dirs and not files: - shutil.rmtree(root) - - for blob in blob_list: - fname = os.path.join(path, blob.name) - update = False - if os.path.exists(fname): - # File exists, check the hashes - source_md5 = blob.properties.content_settings.content_md5 - local_md5 = base64.b64encode( - salt.utils.hashutils.get_hash(fname, "md5").decode("hex") - ) - if local_md5 != source_md5: - update = True - else: - update = True - - if update: - if not os.path.exists(os.path.dirname(fname)): - os.makedirs(os.path.dirname(fname)) - # Lock writes - lk_fn = fname + ".lk" - salt.fileserver.wait_lock(lk_fn, fname) - with salt.utils.files.fopen(lk_fn, "w"): - pass - - try: - blob_service.get_blob_to_path(name, blob.name, fname) - except Exception as exc: # pylint: disable=broad-except - log.exception("Error occurred fetching blob from azurefs") - continue - - # Unlock writes - try: - os.unlink(lk_fn) - except Exception: # pylint: disable=broad-except - pass - - # Write out file list - container_list = path + ".list" - lk_fn = container_list + ".lk" - salt.fileserver.wait_lock(lk_fn, container_list) - with salt.utils.files.fopen(lk_fn, "w"): - pass - with salt.utils.files.fopen(container_list, "w") as fp_: - salt.utils.json.dump(blob_names, fp_) - try: - os.unlink(lk_fn) - except Exception: # pylint: disable=broad-except - pass - try: - hash_cachedir = os.path.join(__opts__["cachedir"], "azurefs", "hashes") - shutil.rmtree(hash_cachedir) - except Exception: # pylint: disable=broad-except - log.exception("Problem occurred trying to invalidate hash cach for azurefs") - - -def file_hash(load, fnd): - """ - Return a file hash based on the hash type set in the master config - """ - if not all(x in load for x in ("path", "saltenv")): - return "", None - ret = {"hash_type": __opts__["hash_type"]} - relpath = fnd["rel"] - path = fnd["path"] - hash_cachedir = os.path.join(__opts__["cachedir"], "azurefs", "hashes") - hashdest = salt.utils.path.join( - hash_cachedir, - load["saltenv"], - "{}.hash.{}".format(relpath, __opts__["hash_type"]), - ) - if not os.path.isfile(hashdest): - if not os.path.exists(os.path.dirname(hashdest)): - os.makedirs(os.path.dirname(hashdest)) - ret["hsum"] = salt.utils.hashutils.get_hash(path, __opts__["hash_type"]) - with salt.utils.files.fopen(hashdest, "w+") as fp_: - fp_.write(salt.utils.stringutils.to_str(ret["hsum"])) - return ret - else: - with salt.utils.files.fopen(hashdest, "rb") as fp_: - ret["hsum"] = salt.utils.stringutils.to_unicode(fp_.read()) - return ret - - -def file_list(load): - """ - Return a list of all files in a specified environment - """ - ret = set() - try: - for container in __opts__["azurefs"]: - if container.get("saltenv", "base") != load["saltenv"]: - continue - container_list = _get_container_path(container) + ".list" - lk = container_list + ".lk" - salt.fileserver.wait_lock(lk, container_list, 5) - if not os.path.exists(container_list): - continue - with salt.utils.files.fopen(container_list, "r") as fp_: - ret.update(set(salt.utils.json.load(fp_))) - except Exception as exc: # pylint: disable=broad-except - log.error( - "azurefs: an error ocurred retrieving file lists. " - "It should be resolved next time the fileserver " - "updates. Please do not manually modify the azurefs " - "cache directory." - ) - return list(ret) - - -def dir_list(load): - """ - Return a list of all directories in a specified environment - """ - ret = set() - files = file_list(load) - for f in files: - dirname = f - while dirname: - dirname = os.path.dirname(dirname) - if dirname: - ret.add(dirname) - return list(ret) - - -def _get_container_path(container): - """ - Get the cache path for the container in question - - Cache paths are generate by combining the account name, container name, - and saltenv, separated by underscores - """ - root = os.path.join(__opts__["cachedir"], "azurefs") - container_dir = "{}_{}_{}".format( - container.get("account_name", ""), - container.get("container_name", ""), - container.get("saltenv", "base"), - ) - return os.path.join(root, container_dir) - - -def _get_container_service(container): - """ - Get the azure block blob service for the container in question - - Try account_key, sas_token, and no auth in that order - """ - if "account_key" in container: - account = azure.storage.CloudStorageAccount( - container["account_name"], account_key=container["account_key"] - ) - elif "sas_token" in container: - account = azure.storage.CloudStorageAccount( - container["account_name"], sas_token=container["sas_token"] - ) - else: - account = azure.storage.CloudStorageAccount(container["account_name"]) - blob_service = account.create_block_blob_service() - return blob_service - - -def _validate_config(): - """ - Validate azurefs config, return False if it doesn't validate - """ - if not isinstance(__opts__["azurefs"], list): - log.error("azurefs configuration is not formed as a list, skipping azurefs") - return False - for container in __opts__["azurefs"]: - if not isinstance(container, dict): - log.error( - "One or more entries in the azurefs configuration list are " - "not formed as a dict. Skipping azurefs: %s", - container, - ) - return False - if "account_name" not in container or "container_name" not in container: - log.error( - "An azurefs container configuration is missing either an " - "account_name or a container_name: %s", - container, - ) - return False - return True diff --git a/salt/grains/metadata_azure.py b/salt/grains/metadata_azure.py deleted file mode 100644 index 4cdf8243473..00000000000 --- a/salt/grains/metadata_azure.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Grains from cloud metadata servers at 169.254.169.254 in Azure Virtual Machine - -.. versionadded:: 3006.0 - -:depends: requests - -To enable these grains that pull from the http://169.254.169.254/metadata/instance?api-version=2020-09-01 -metadata server set `metadata_server_grains: True` in the minion config. - -.. code-block:: yaml - - metadata_server_grains: True - -""" - -import logging - -import salt.utils.http as http -import salt.utils.json - -HOST = "http://169.254.169.254" -URL = f"{HOST}/metadata/instance?api-version=2020-09-01" -log = logging.getLogger(__name__) - - -def __virtual__(): - # Check if metadata_server_grains minion option is enabled - if __opts__.get("metadata_server_grains", False) is False: - return False - azuretest = http.query( - URL, status=True, headers=True, header_list=["Metadata: true"] - ) - if azuretest.get("status", 404) != 200: - return False - return True - - -def metadata(): - """Takes no arguments, returns a dictionary of metadata values from Azure.""" - log.debug("All checks true - loading azure metadata") - result = http.query(URL, headers=True, header_list=["Metadata: true"]) - metadata = salt.utils.json.loads(result.get("body", {})) - - return metadata diff --git a/salt/modules/azurearm_compute.py b/salt/modules/azurearm_compute.py deleted file mode 100644 index 61ff7f85b2f..00000000000 --- a/salt/modules/azurearm_compute.py +++ /dev/null @@ -1,754 +0,0 @@ -""" -Azure (ARM) Compute Execution Module - -.. versionadded:: 2019.2.0 - -.. warning:: - - This cloud provider will be removed from Salt in version 3007 in favor of - the `saltext.azurerm Salt Extension - `_ - -:maintainer: -:maturity: new -:depends: - * `azure `_ >= 2.0.0 - * `azure-common `_ >= 1.1.8 - * `azure-mgmt `_ >= 1.0.0 - * `azure-mgmt-compute `_ >= 1.0.0 - * `azure-mgmt-network `_ >= 1.7.1 - * `azure-mgmt-resource `_ >= 1.1.0 - * `azure-mgmt-storage `_ >= 1.0.0 - * `azure-mgmt-web `_ >= 0.32.0 - * `azure-storage `_ >= 0.34.3 - * `msrestazure `_ >= 0.4.21 -:platform: linux - -:configuration: This module requires Azure Resource Manager credentials to be passed as keyword arguments - to every function in order to work properly. - - Required provider parameters: - - if using username and password: - * ``subscription_id`` - * ``username`` - * ``password`` - - if using a service principal: - * ``subscription_id`` - * ``tenant`` - * ``client_id`` - * ``secret`` - - Optional provider parameters: - -**cloud_environment**: Used to point the cloud driver to different API endpoints, such as Azure GovCloud. - Possible values: - * ``AZURE_PUBLIC_CLOUD`` (default) - * ``AZURE_CHINA_CLOUD`` - * ``AZURE_US_GOV_CLOUD`` - * ``AZURE_GERMAN_CLOUD`` - -""" - -# Python libs - -import logging -from functools import wraps - -import salt.utils.azurearm - -# Azure libs -HAS_LIBS = False -try: - import azure.mgmt.compute.models # pylint: disable=unused-import - from msrest.exceptions import SerializationError - from msrestazure.azure_exceptions import CloudError - - HAS_LIBS = True -except ImportError: - pass - -__virtualname__ = "azurearm_compute" - -log = logging.getLogger(__name__) - - -def __virtual__(): - if not HAS_LIBS: - return ( - False, - "The following dependencies are required to use the AzureARM modules: " - "Microsoft Azure SDK for Python >= 2.0rc6, " - "MS REST Azure (msrestazure) >= 0.4", - ) - - return __virtualname__ - - -def _deprecation_message(function): - """ - Decorator wrapper to warn about azurearm deprecation - """ - - @wraps(function) - def wrapped(*args, **kwargs): - salt.utils.versions.warn_until( - "Chlorine", - "The 'azurearm' functionality in Salt has been deprecated and its " - "functionality will be removed in version 3007 in favor of the " - "saltext.azurerm Salt Extension. " - "(https://github.com/salt-extensions/saltext-azurerm)", - category=FutureWarning, - ) - ret = function(*args, **salt.utils.args.clean_kwargs(**kwargs)) - return ret - - return wrapped - - -@_deprecation_message -def availability_set_create_or_update( - name, resource_group, **kwargs -): # pylint: disable=invalid-name - """ - .. versionadded:: 2019.2.0 - - Create or update an availability set. - - :param name: The availability set to create. - - :param resource_group: The resource group name assigned to the - availability set. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.availability_set_create_or_update testset testgroup - - """ - if "location" not in kwargs: - rg_props = __salt__["azurearm_resource.resource_group_get"]( - resource_group, **kwargs - ) - - if "error" in rg_props: - log.error("Unable to determine location from resource group specified.") - return False - kwargs["location"] = rg_props["location"] - - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - - # Use VM names to link to the IDs of existing VMs. - if isinstance(kwargs.get("virtual_machines"), list): - vm_list = [] - for vm_name in kwargs.get("virtual_machines"): - vm_instance = __salt__["azurearm_compute.virtual_machine_get"]( - name=vm_name, resource_group=resource_group, **kwargs - ) - if "error" not in vm_instance: - vm_list.append({"id": str(vm_instance["id"])}) - kwargs["virtual_machines"] = vm_list - - try: - setmodel = __utils__["azurearm.create_object_model"]( - "compute", "AvailabilitySet", **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - av_set = compconn.availability_sets.create_or_update( - resource_group_name=resource_group, - availability_set_name=name, - parameters=setmodel, - ) - result = av_set.as_dict() - - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def availability_set_delete(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete an availability set. - - :param name: The availability set to delete. - - :param resource_group: The resource group name assigned to the - availability set. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.availability_set_delete testset testgroup - - """ - result = False - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - compconn.availability_sets.delete( - resource_group_name=resource_group, availability_set_name=name - ) - result = True - - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - - return result - - -@_deprecation_message -def availability_set_get(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get a dictionary representing an availability set's properties. - - :param name: The availability set to get. - - :param resource_group: The resource group name assigned to the - availability set. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.availability_set_get testset testgroup - - """ - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - av_set = compconn.availability_sets.get( - resource_group_name=resource_group, availability_set_name=name - ) - result = av_set.as_dict() - - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def availability_sets_list(resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all availability sets within a resource group. - - :param resource_group: The resource group name to list availability - sets within. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.availability_sets_list testgroup - - """ - result = {} - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - avail_sets = __utils__["azurearm.paged_object_to_list"]( - compconn.availability_sets.list(resource_group_name=resource_group) - ) - - for avail_set in avail_sets: - result[avail_set["name"]] = avail_set - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def availability_sets_list_available_sizes( - name, resource_group, **kwargs -): # pylint: disable=invalid-name - """ - .. versionadded:: 2019.2.0 - - List all available virtual machine sizes that can be used to - to create a new virtual machine in an existing availability set. - - :param name: The availability set name to list available - virtual machine sizes within. - - :param resource_group: The resource group name to list available - availability set sizes within. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.availability_sets_list_available_sizes testset testgroup - - """ - result = {} - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - sizes = __utils__["azurearm.paged_object_to_list"]( - compconn.availability_sets.list_available_sizes( - resource_group_name=resource_group, availability_set_name=name - ) - ) - - for size in sizes: - result[size["name"]] = size - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def virtual_machine_capture( - name, destination_name, resource_group, prefix="capture-", overwrite=False, **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Captures the VM by copying virtual hard disks of the VM and outputs - a template that can be used to create similar VMs. - - :param name: The name of the virtual machine. - - :param destination_name: The destination container name. - - :param resource_group: The resource group name assigned to the - virtual machine. - - :param prefix: (Default: 'capture-') The captured virtual hard disk's name prefix. - - :param overwrite: (Default: False) Overwrite the destination disk in case of conflict. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.virtual_machine_capture testvm testcontainer testgroup - - """ - # pylint: disable=invalid-name - VirtualMachineCaptureParameters = getattr( - azure.mgmt.compute.models, "VirtualMachineCaptureParameters" - ) - - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - # pylint: disable=invalid-name - vm = compconn.virtual_machines.capture( - resource_group_name=resource_group, - vm_name=name, - parameters=VirtualMachineCaptureParameters( - vhd_prefix=prefix, - destination_container_name=destination_name, - overwrite_vhds=overwrite, - ), - ) - vm.wait() - vm_result = vm.result() - result = vm_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def virtual_machine_get(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Retrieves information about the model view or the instance view of a - virtual machine. - - :param name: The name of the virtual machine. - - :param resource_group: The resource group name assigned to the - virtual machine. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.virtual_machine_get testvm testgroup - - """ - expand = kwargs.get("expand") - - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - # pylint: disable=invalid-name - vm = compconn.virtual_machines.get( - resource_group_name=resource_group, vm_name=name, expand=expand - ) - result = vm.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def virtual_machine_convert_to_managed_disks( - name, resource_group, **kwargs -): # pylint: disable=invalid-name - """ - .. versionadded:: 2019.2.0 - - Converts virtual machine disks from blob-based to managed disks. Virtual - machine must be stop-deallocated before invoking this operation. - - :param name: The name of the virtual machine to convert. - - :param resource_group: The resource group name assigned to the - virtual machine. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.virtual_machine_convert_to_managed_disks testvm testgroup - - """ - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - # pylint: disable=invalid-name - vm = compconn.virtual_machines.convert_to_managed_disks( - resource_group_name=resource_group, vm_name=name - ) - vm.wait() - vm_result = vm.result() - result = vm_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def virtual_machine_deallocate(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Power off a virtual machine and deallocate compute resources. - - :param name: The name of the virtual machine to deallocate. - - :param resource_group: The resource group name assigned to the - virtual machine. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.virtual_machine_deallocate testvm testgroup - - """ - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - # pylint: disable=invalid-name - vm = compconn.virtual_machines.deallocate( - resource_group_name=resource_group, vm_name=name - ) - vm.wait() - vm_result = vm.result() - result = vm_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def virtual_machine_generalize(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Set the state of a virtual machine to 'generalized'. - - :param name: The name of the virtual machine. - - :param resource_group: The resource group name assigned to the - virtual machine. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.virtual_machine_generalize testvm testgroup - - """ - result = False - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - compconn.virtual_machines.generalize( - resource_group_name=resource_group, vm_name=name - ) - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - - return result - - -@_deprecation_message -def virtual_machines_list(resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all virtual machines within a resource group. - - :param resource_group: The resource group name to list virtual - machines within. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.virtual_machines_list testgroup - - """ - result = {} - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - vms = __utils__["azurearm.paged_object_to_list"]( - compconn.virtual_machines.list(resource_group_name=resource_group) - ) - for vm in vms: # pylint: disable=invalid-name - result[vm["name"]] = vm - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def virtual_machines_list_all(**kwargs): - """ - .. versionadded:: 2019.2.0 - - List all virtual machines within a subscription. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.virtual_machines_list_all - - """ - result = {} - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - vms = __utils__["azurearm.paged_object_to_list"]( - compconn.virtual_machines.list_all() - ) - for vm in vms: # pylint: disable=invalid-name - result[vm["name"]] = vm - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def virtual_machines_list_available_sizes( - name, resource_group, **kwargs -): # pylint: disable=invalid-name - """ - .. versionadded:: 2019.2.0 - - Lists all available virtual machine sizes to which the specified virtual - machine can be resized. - - :param name: The name of the virtual machine. - - :param resource_group: The resource group name assigned to the - virtual machine. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.virtual_machines_list_available_sizes testvm testgroup - - """ - result = {} - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - sizes = __utils__["azurearm.paged_object_to_list"]( - compconn.virtual_machines.list_available_sizes( - resource_group_name=resource_group, vm_name=name - ) - ) - for size in sizes: - result[size["name"]] = size - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def virtual_machine_power_off(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Power off (stop) a virtual machine. - - :param name: The name of the virtual machine to stop. - - :param resource_group: The resource group name assigned to the - virtual machine. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.virtual_machine_power_off testvm testgroup - - """ - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - # pylint: disable=invalid-name - vm = compconn.virtual_machines.power_off( - resource_group_name=resource_group, vm_name=name - ) - vm.wait() - vm_result = vm.result() - result = vm_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def virtual_machine_restart(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Restart a virtual machine. - - :param name: The name of the virtual machine to restart. - - :param resource_group: The resource group name assigned to the - virtual machine. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.virtual_machine_restart testvm testgroup - - """ - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - # pylint: disable=invalid-name - vm = compconn.virtual_machines.restart( - resource_group_name=resource_group, vm_name=name - ) - vm.wait() - vm_result = vm.result() - result = vm_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def virtual_machine_start(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Power on (start) a virtual machine. - - :param name: The name of the virtual machine to start. - - :param resource_group: The resource group name assigned to the - virtual machine. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.virtual_machine_start testvm testgroup - - """ - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - # pylint: disable=invalid-name - vm = compconn.virtual_machines.start( - resource_group_name=resource_group, vm_name=name - ) - vm.wait() - vm_result = vm.result() - result = vm_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def virtual_machine_redeploy(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Redeploy a virtual machine. - - :param name: The name of the virtual machine to redeploy. - - :param resource_group: The resource group name assigned to the - virtual machine. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_compute.virtual_machine_redeploy testvm testgroup - - """ - compconn = __utils__["azurearm.get_client"]("compute", **kwargs) - try: - # pylint: disable=invalid-name - vm = compconn.virtual_machines.redeploy( - resource_group_name=resource_group, vm_name=name - ) - vm.wait() - vm_result = vm.result() - result = vm_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("compute", str(exc), **kwargs) - result = {"error": str(exc)} - - return result diff --git a/salt/modules/azurearm_dns.py b/salt/modules/azurearm_dns.py deleted file mode 100644 index e503712f264..00000000000 --- a/salt/modules/azurearm_dns.py +++ /dev/null @@ -1,552 +0,0 @@ -""" -Azure (ARM) DNS Execution Module - -.. versionadded:: 3000 - -.. warning:: - - This cloud provider will be removed from Salt in version 3007 in favor of - the `saltext.azurerm Salt Extension - `_ - -:maintainer: -:maturity: new -:depends: - * `azure `_ >= 2.0.0 - * `azure-common `_ >= 1.1.8 - * `azure-mgmt `_ >= 1.0.0 - * `azure-mgmt-compute `_ >= 1.0.0 - * `azure-mgmt-dns `_ >= 2.0.0rc1 - * `azure-mgmt-network `_ >= 1.7.1 - * `azure-mgmt-resource `_ >= 1.1.0 - * `azure-mgmt-storage `_ >= 1.0.0 - * `azure-mgmt-web `_ >= 0.32.0 - * `azure-storage `_ >= 0.34.3 - * `msrestazure `_ >= 0.4.21 - -:platform: linux -:configuration: - This module requires Azure Resource Manager credentials to be passed as keyword arguments - to every function in order to work properly. - -Required provider parameters: - - if using username and password: - - * ``subscription_id`` - * ``username`` - * ``password`` - - if using a service principal: - - * ``subscription_id`` - * ``tenant`` - * ``client_id`` - * ``secret`` - -Optional provider parameters: - - **cloud_environment**: Used to point the cloud driver to different API endpoints, such as Azure GovCloud. - - Possible values: - - * ``AZURE_PUBLIC_CLOUD`` (default) - * ``AZURE_CHINA_CLOUD`` - * ``AZURE_US_GOV_CLOUD`` - * ``AZURE_GERMAN_CLOUD`` - -""" - -# Python libs - -import logging -from functools import wraps - -import salt.utils.azurearm - -# Azure libs -HAS_LIBS = False -try: - import azure.mgmt.dns.models # pylint: disable=unused-import - from msrest.exceptions import SerializationError - from msrestazure.azure_exceptions import CloudError - - HAS_LIBS = True -except ImportError: - pass - -__virtualname__ = "azurearm_dns" - -log = logging.getLogger(__name__) - - -def __virtual__(): - if not HAS_LIBS: - return ( - False, - "The following dependencies are required to use the AzureARM modules: " - "Microsoft Azure SDK for Python >= 2.0rc6, " - "MS REST Azure (msrestazure) >= 0.4", - ) - - return __virtualname__ - - -def _deprecation_message(function): - """ - Decorator wrapper to warn about azurearm deprecation - """ - - @wraps(function) - def wrapped(*args, **kwargs): - salt.utils.versions.warn_until( - "Chlorine", - "The 'azurearm' functionality in Salt has been deprecated and its " - "functionality will be removed in version 3007 in favor of the " - "saltext.azurerm Salt Extension. " - "(https://github.com/salt-extensions/saltext-azurerm)", - category=FutureWarning, - ) - ret = function(*args, **salt.utils.args.clean_kwargs(**kwargs)) - return ret - - return wrapped - - -@_deprecation_message -def record_set_create_or_update(name, zone_name, resource_group, record_type, **kwargs): - """ - .. versionadded:: 3000 - - Creates or updates a record set within a DNS zone. - - :param name: The name of the record set, relative to the name of the zone. - - :param zone_name: The name of the DNS zone (without a terminating dot). - - :param resource_group: The name of the resource group. - - :param record_type: - The type of DNS record in this record set. Record sets of type SOA can be - updated but not created (they are created when the DNS zone is created). - Possible values include: 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SOA', 'SRV', 'TXT' - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_dns.record_set_create_or_update myhost myzone testgroup A - arecords='[{ipv4_address: 10.0.0.1}]' ttl=300 - - """ - dnsconn = __utils__["azurearm.get_client"]("dns", **kwargs) - - try: - record_set_model = __utils__["azurearm.create_object_model"]( - "dns", "RecordSet", **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - record_set = dnsconn.record_sets.create_or_update( - relative_record_set_name=name, - zone_name=zone_name, - resource_group_name=resource_group, - record_type=record_type, - parameters=record_set_model, - if_match=kwargs.get("if_match"), - if_none_match=kwargs.get("if_none_match"), - ) - result = record_set.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("dns", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def record_set_delete(name, zone_name, resource_group, record_type, **kwargs): - """ - .. versionadded:: 3000 - - Deletes a record set from a DNS zone. This operation cannot be undone. - - :param name: The name of the record set, relative to the name of the zone. - - :param zone_name: The name of the DNS zone (without a terminating dot). - - :param resource_group: The name of the resource group. - - :param record_type: - The type of DNS record in this record set. Record sets of type SOA cannot be - deleted (they are deleted when the DNS zone is deleted). - Possible values include: 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SOA', 'SRV', 'TXT' - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_dns.record_set_delete myhost myzone testgroup A - - """ - result = False - dnsconn = __utils__["azurearm.get_client"]("dns", **kwargs) - try: - record_set = dnsconn.record_sets.delete( - relative_record_set_name=name, - zone_name=zone_name, - resource_group_name=resource_group, - record_type=record_type, - if_match=kwargs.get("if_match"), - ) - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("dns", str(exc), **kwargs) - - return result - - -@_deprecation_message -def record_set_get(name, zone_name, resource_group, record_type, **kwargs): - """ - .. versionadded:: 3000 - - Get a dictionary representing a record set's properties. - - :param name: The name of the record set, relative to the name of the zone. - - :param zone_name: The name of the DNS zone (without a terminating dot). - - :param resource_group: The name of the resource group. - - :param record_type: - The type of DNS record in this record set. - Possible values include: 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SOA', 'SRV', 'TXT' - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_dns.record_set_get '@' myzone testgroup SOA - - """ - dnsconn = __utils__["azurearm.get_client"]("dns", **kwargs) - try: - record_set = dnsconn.record_sets.get( - relative_record_set_name=name, - zone_name=zone_name, - resource_group_name=resource_group, - record_type=record_type, - ) - result = record_set.as_dict() - - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("dns", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def record_sets_list_by_type( - zone_name, resource_group, record_type, top=None, recordsetnamesuffix=None, **kwargs -): - """ - .. versionadded:: 3000 - - Lists the record sets of a specified type in a DNS zone. - - :param zone_name: The name of the DNS zone (without a terminating dot). - - :param resource_group: The name of the resource group. - - :param record_type: - The type of record sets to enumerate. - Possible values include: 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SOA', 'SRV', 'TXT' - - :param top: - The maximum number of record sets to return. If not specified, - returns up to 100 record sets. - - :param recordsetnamesuffix: - The suffix label of the record set name that has - to be used to filter the record set enumerations. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_dns.record_sets_list_by_type myzone testgroup SOA - - """ - result = {} - dnsconn = __utils__["azurearm.get_client"]("dns", **kwargs) - try: - record_sets = __utils__["azurearm.paged_object_to_list"]( - dnsconn.record_sets.list_by_type( - zone_name=zone_name, - resource_group_name=resource_group, - record_type=record_type, - top=top, - recordsetnamesuffix=recordsetnamesuffix, - ) - ) - - for record_set in record_sets: - result[record_set["name"]] = record_set - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("dns", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def record_sets_list_by_dns_zone( - zone_name, resource_group, top=None, recordsetnamesuffix=None, **kwargs -): - """ - .. versionadded:: 3000 - - Lists all record sets in a DNS zone. - - :param zone_name: The name of the DNS zone (without a terminating dot). - - :param resource_group: The name of the resource group. - - :param top: - The maximum number of record sets to return. If not specified, - returns up to 100 record sets. - - :param recordsetnamesuffix: - The suffix label of the record set name that has - to be used to filter the record set enumerations. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_dns.record_sets_list_by_dns_zone myzone testgroup - - """ - result = {} - dnsconn = __utils__["azurearm.get_client"]("dns", **kwargs) - try: - record_sets = __utils__["azurearm.paged_object_to_list"]( - dnsconn.record_sets.list_by_dns_zone( - zone_name=zone_name, - resource_group_name=resource_group, - top=top, - recordsetnamesuffix=recordsetnamesuffix, - ) - ) - - for record_set in record_sets: - result[record_set["name"]] = record_set - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("dns", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def zone_create_or_update(name, resource_group, **kwargs): - """ - .. versionadded:: 3000 - - Creates or updates a DNS zone. Does not modify DNS records within the zone. - - :param name: The name of the DNS zone to create (without a terminating dot). - - :param resource_group: The name of the resource group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_dns.zone_create_or_update myzone testgroup - - """ - # DNS zones are global objects - kwargs["location"] = "global" - - dnsconn = __utils__["azurearm.get_client"]("dns", **kwargs) - - # Convert list of ID strings to list of dictionaries with id key. - if isinstance(kwargs.get("registration_virtual_networks"), list): - kwargs["registration_virtual_networks"] = [ - {"id": vnet} for vnet in kwargs["registration_virtual_networks"] - ] - - if isinstance(kwargs.get("resolution_virtual_networks"), list): - kwargs["resolution_virtual_networks"] = [ - {"id": vnet} for vnet in kwargs["resolution_virtual_networks"] - ] - - try: - zone_model = __utils__["azurearm.create_object_model"]("dns", "Zone", **kwargs) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - zone = dnsconn.zones.create_or_update( - zone_name=name, - resource_group_name=resource_group, - parameters=zone_model, - if_match=kwargs.get("if_match"), - if_none_match=kwargs.get("if_none_match"), - ) - result = zone.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("dns", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def zone_delete(name, resource_group, **kwargs): - """ - .. versionadded:: 3000 - - Delete a DNS zone within a resource group. - - :param name: The name of the DNS zone to delete. - - :param resource_group: The name of the resource group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_dns.zone_delete myzone testgroup - - """ - result = False - dnsconn = __utils__["azurearm.get_client"]("dns", **kwargs) - try: - zone = dnsconn.zones.delete( - zone_name=name, - resource_group_name=resource_group, - if_match=kwargs.get("if_match"), - ) - zone.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("dns", str(exc), **kwargs) - - return result - - -@_deprecation_message -def zone_get(name, resource_group, **kwargs): - """ - .. versionadded:: 3000 - - Get a dictionary representing a DNS zone's properties, but not the - record sets within the zone. - - :param name: The DNS zone to get. - - :param resource_group: The name of the resource group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_dns.zone_get myzone testgroup - - """ - dnsconn = __utils__["azurearm.get_client"]("dns", **kwargs) - try: - zone = dnsconn.zones.get(zone_name=name, resource_group_name=resource_group) - result = zone.as_dict() - - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("dns", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def zones_list_by_resource_group(resource_group, top=None, **kwargs): - """ - .. versionadded:: 3000 - - Lists the DNS zones in a resource group. - - :param resource_group: The name of the resource group. - - :param top: - The maximum number of DNS zones to return. If not specified, - returns up to 100 zones. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_dns.zones_list_by_resource_group testgroup - - """ - result = {} - dnsconn = __utils__["azurearm.get_client"]("dns", **kwargs) - try: - zones = __utils__["azurearm.paged_object_to_list"]( - dnsconn.zones.list_by_resource_group( - resource_group_name=resource_group, top=top - ) - ) - - for zone in zones: - result[zone["name"]] = zone - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("dns", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def zones_list(top=None, **kwargs): - """ - .. versionadded:: 3000 - - Lists the DNS zones in all resource groups in a subscription. - - :param top: - The maximum number of DNS zones to return. If not specified, - eturns up to 100 zones. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_dns.zones_list - - """ - result = {} - dnsconn = __utils__["azurearm.get_client"]("dns", **kwargs) - try: - zones = __utils__["azurearm.paged_object_to_list"](dnsconn.zones.list(top=top)) - - for zone in zones: - result[zone["name"]] = zone - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("dns", str(exc), **kwargs) - result = {"error": str(exc)} - - return result diff --git a/salt/modules/azurearm_network.py b/salt/modules/azurearm_network.py deleted file mode 100644 index c8e520c3cf2..00000000000 --- a/salt/modules/azurearm_network.py +++ /dev/null @@ -1,2859 +0,0 @@ -""" -Azure (ARM) Network Execution Module - -.. versionadded:: 2019.2.0 - -.. warning:: - - This cloud provider will be removed from Salt in version 3007 in favor of - the `saltext.azurerm Salt Extension - `_ - -:maintainer: -:maturity: new -:depends: - * `azure `_ >= 2.0.0 - * `azure-common `_ >= 1.1.8 - * `azure-mgmt `_ >= 1.0.0 - * `azure-mgmt-compute `_ >= 1.0.0 - * `azure-mgmt-network `_ >= 1.7.1 - * `azure-mgmt-resource `_ >= 1.1.0 - * `azure-mgmt-storage `_ >= 1.0.0 - * `azure-mgmt-web `_ >= 0.32.0 - * `azure-storage `_ >= 0.34.3 - * `msrestazure `_ >= 0.4.21 -:platform: linux - -:configuration: This module requires Azure Resource Manager credentials to be passed as keyword arguments - to every function in order to work properly. - - Required provider parameters: - - if using username and password: - * ``subscription_id`` - * ``username`` - * ``password`` - - if using a service principal: - * ``subscription_id`` - * ``tenant`` - * ``client_id`` - * ``secret`` - - Optional provider parameters: - -**cloud_environment**: Used to point the cloud driver to different API endpoints, such as Azure GovCloud. - Possible values: - * ``AZURE_PUBLIC_CLOUD`` (default) - * ``AZURE_CHINA_CLOUD`` - * ``AZURE_US_GOV_CLOUD`` - * ``AZURE_GERMAN_CLOUD`` - -""" - -# Python libs - -import logging -from functools import wraps - -# Salt libs -import salt.utils.azurearm -from salt.exceptions import SaltInvocationError # pylint: disable=unused-import - -# Azure libs -HAS_LIBS = False -try: - import azure.mgmt.network.models # pylint: disable=unused-import - from msrest.exceptions import SerializationError - from msrestazure.azure_exceptions import CloudError - - HAS_LIBS = True -except ImportError: - pass - -__virtualname__ = "azurearm_network" - -log = logging.getLogger(__name__) - - -def __virtual__(): - if not HAS_LIBS: - return ( - False, - "The following dependencies are required to use the AzureARM modules: " - "Microsoft Azure SDK for Python >= 2.0rc6, " - "MS REST Azure (msrestazure) >= 0.4", - ) - - return __virtualname__ - - -def _deprecation_message(function): - """ - Decorator wrapper to warn about azurearm deprecation - """ - - @wraps(function) - def wrapped(*args, **kwargs): - salt.utils.versions.warn_until( - "Chlorine", - "The 'azurearm' functionality in Salt has been deprecated and its " - "functionality will be removed in version 3007 in favor of the " - "saltext.azurerm Salt Extension. " - "(https://github.com/salt-extensions/saltext-azurerm)", - category=FutureWarning, - ) - ret = function(*args, **salt.utils.args.clean_kwargs(**kwargs)) - return ret - - return wrapped - - -@_deprecation_message -def check_dns_name_availability(name, region, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Check whether a domain name in the current zone is available for use. - - :param name: The DNS name to query. - - :param region: The region to query for the DNS name in question. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.check_dns_name_availability testdnsname westus - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - check_dns_name = netconn.check_dns_name_availability( - location=region, domain_name_label=name - ) - result = check_dns_name.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def check_ip_address_availability( - ip_address, virtual_network, resource_group, **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Check that a private ip address is available within the specified - virtual network. - - :param ip_address: The ip_address to query. - - :param virtual_network: The virtual network to query for the IP address - in question. - - :param resource_group: The resource group name assigned to the - virtual network. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.check_ip_address_availability 10.0.0.4 testnet testgroup - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - check_ip = netconn.virtual_networks.check_ip_address_availability( - resource_group_name=resource_group, - virtual_network_name=virtual_network, - ip_address=ip_address, - ) - result = check_ip.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def default_security_rule_get(name, security_group, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a default security rule within a security group. - - :param name: The name of the security rule to query. - - :param security_group: The network security group containing the - security rule. - - :param resource_group: The resource group name assigned to the - network security group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.default_security_rule_get DenyAllOutBound testnsg testgroup - - """ - result = {} - - default_rules = default_security_rules_list( - security_group=security_group, resource_group=resource_group, **kwargs - ) - - if isinstance(default_rules, dict) and "error" in default_rules: - return default_rules - - try: - for default_rule in default_rules: - if default_rule["name"] == name: - result = default_rule - if not result: - result = {"error": "Unable to find {} in {}!".format(name, security_group)} - except KeyError as exc: - log.error("Unable to find %s in %s!", name, security_group) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def default_security_rules_list(security_group, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List default security rules within a security group. - - :param security_group: The network security group to query. - - :param resource_group: The resource group name assigned to the - network security group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.default_security_rules_list testnsg testgroup - - """ - result = {} - - secgroup = network_security_group_get( - security_group=security_group, resource_group=resource_group, **kwargs - ) - - if "error" in secgroup: - return secgroup - - try: - result = secgroup["default_security_rules"] - except KeyError as exc: - log.error("No default security rules found for %s!", security_group) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def security_rules_list(security_group, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List security rules within a network security group. - - :param security_group: The network security group to query. - - :param resource_group: The resource group name assigned to the - network security group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.security_rules_list testnsg testgroup - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - secrules = netconn.security_rules.list( - network_security_group_name=security_group, - resource_group_name=resource_group, - ) - result = __utils__["azurearm.paged_object_to_list"](secrules) - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def security_rule_create_or_update( - name, - access, - direction, - priority, - protocol, - security_group, - resource_group, - source_address_prefix=None, - destination_address_prefix=None, - source_port_range=None, - destination_port_range=None, - source_address_prefixes=None, - destination_address_prefixes=None, - source_port_ranges=None, - destination_port_ranges=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Create or update a security rule within a specified network security group. - - :param name: The name of the security rule to create. - - :param access: - 'allow' or 'deny' - - :param direction: - 'inbound' or 'outbound' - - :param priority: - Integer between 100 and 4096 used for ordering rule application. - - :param protocol: - 'tcp', 'udp', or '*' - - :param destination_address_prefix: - The CIDR or destination IP range. Asterix '*' can also be used to match all destination IPs. - Default tags such as 'VirtualNetwork', 'AzureLoadBalancer' and 'Internet' can also be used. - If this is an ingress rule, specifies where network traffic originates from. - - :param destination_port_range: - The destination port or range. Integer or range between 0 and 65535. Asterix '*' - can also be used to match all ports. - - :param source_address_prefix: - The CIDR or source IP range. Asterix '*' can also be used to match all source IPs. - Default tags such as 'VirtualNetwork', 'AzureLoadBalancer' and 'Internet' can also be used. - If this is an ingress rule, specifies where network traffic originates from. - - :param source_port_range: - The source port or range. Integer or range between 0 and 65535. Asterix '*' - can also be used to match all ports. - - :param destination_address_prefixes: - A list of destination_address_prefix values. This parameter overrides destination_address_prefix - and will cause any value entered there to be ignored. - - :param destination_port_ranges: - A list of destination_port_range values. This parameter overrides destination_port_range - and will cause any value entered there to be ignored. - - :param source_address_prefixes: - A list of source_address_prefix values. This parameter overrides source_address_prefix - and will cause any value entered there to be ignored. - - :param source_port_ranges: - A list of source_port_range values. This parameter overrides source_port_range - and will cause any value entered there to be ignored. - - :param security_group: The network security group containing the - security rule. - - :param resource_group: The resource group name assigned to the - network security group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.security_rule_create_or_update testrule1 allow outbound 101 tcp testnsg testgroup \ - source_address_prefix='*' destination_address_prefix=internet source_port_range='*' \ - destination_port_range='1-1024' - - """ - exclusive_params = [ - ("source_port_ranges", "source_port_range"), - ("source_address_prefixes", "source_address_prefix"), - ("destination_port_ranges", "destination_port_range"), - ("destination_address_prefixes", "destination_address_prefix"), - ] - - for params in exclusive_params: - # pylint: disable=eval-used - if not eval(params[0]) and not eval(params[1]): - log.error( - "Either the %s or %s parameter must be provided!", params[0], params[1] - ) - return False - # pylint: disable=eval-used - if eval(params[0]): - # pylint: disable=exec-used - exec("{} = None".format(params[1])) - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - try: - rulemodel = __utils__["azurearm.create_object_model"]( - "network", - "SecurityRule", - name=name, - access=access, - direction=direction, - priority=priority, - protocol=protocol, - source_port_ranges=source_port_ranges, - source_port_range=source_port_range, - source_address_prefixes=source_address_prefixes, - source_address_prefix=source_address_prefix, - destination_port_ranges=destination_port_ranges, - destination_port_range=destination_port_range, - destination_address_prefixes=destination_address_prefixes, - destination_address_prefix=destination_address_prefix, - **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - secrule = netconn.security_rules.create_or_update( - resource_group_name=resource_group, - network_security_group_name=security_group, - security_rule_name=name, - security_rule_parameters=rulemodel, - ) - secrule.wait() - secrule_result = secrule.result() - result = secrule_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def security_rule_delete(security_rule, security_group, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a security rule within a specified security group. - - :param name: The name of the security rule to delete. - - :param security_group: The network security group containing the - security rule. - - :param resource_group: The resource group name assigned to the - network security group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.security_rule_delete testrule1 testnsg testgroup - - """ - result = False - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - secrule = netconn.security_rules.delete( - network_security_group_name=security_group, - resource_group_name=resource_group, - security_rule_name=security_rule, - ) - secrule.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - - return result - - -@_deprecation_message -def security_rule_get(security_rule, security_group, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get a security rule within a specified network security group. - - :param name: The name of the security rule to query. - - :param security_group: The network security group containing the - security rule. - - :param resource_group: The resource group name assigned to the - network security group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.security_rule_get testrule1 testnsg testgroup - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - secrule = netconn.security_rules.get( - network_security_group_name=security_group, - resource_group_name=resource_group, - security_rule_name=security_rule, - ) - result = secrule.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def network_security_group_create_or_update( - name, resource_group, **kwargs -): # pylint: disable=invalid-name - """ - .. versionadded:: 2019.2.0 - - Create or update a network security group. - - :param name: The name of the network security group to create. - - :param resource_group: The resource group name assigned to the - network security group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.network_security_group_create_or_update testnsg testgroup - - """ - if "location" not in kwargs: - rg_props = __salt__["azurearm_resource.resource_group_get"]( - resource_group, **kwargs - ) - - if "error" in rg_props: - log.error("Unable to determine location from resource group specified.") - return False - kwargs["location"] = rg_props["location"] - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - try: - secgroupmodel = __utils__["azurearm.create_object_model"]( - "network", "NetworkSecurityGroup", **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - secgroup = netconn.network_security_groups.create_or_update( - resource_group_name=resource_group, - network_security_group_name=name, - parameters=secgroupmodel, - ) - secgroup.wait() - secgroup_result = secgroup.result() - result = secgroup_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def network_security_group_delete(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a network security group within a resource group. - - :param name: The name of the network security group to delete. - - :param resource_group: The resource group name assigned to the - network security group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.network_security_group_delete testnsg testgroup - - """ - result = False - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - secgroup = netconn.network_security_groups.delete( - resource_group_name=resource_group, network_security_group_name=name - ) - secgroup.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - - return result - - -@_deprecation_message -def network_security_group_get(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a network security group within a resource group. - - :param name: The name of the network security group to query. - - :param resource_group: The resource group name assigned to the - network security group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.network_security_group_get testnsg testgroup - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - secgroup = netconn.network_security_groups.get( - resource_group_name=resource_group, network_security_group_name=name - ) - result = secgroup.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def network_security_groups_list(resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all network security groups within a resource group. - - :param resource_group: The resource group name to list network security \ - groups within. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.network_security_groups_list testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - secgroups = __utils__["azurearm.paged_object_to_list"]( - netconn.network_security_groups.list(resource_group_name=resource_group) - ) - for secgroup in secgroups: - result[secgroup["name"]] = secgroup - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def network_security_groups_list_all(**kwargs): # pylint: disable=invalid-name - """ - .. versionadded:: 2019.2.0 - - List all network security groups within a subscription. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.network_security_groups_list_all - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - secgroups = __utils__["azurearm.paged_object_to_list"]( - netconn.network_security_groups.list_all() - ) - for secgroup in secgroups: - result[secgroup["name"]] = secgroup - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def subnets_list(virtual_network, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all subnets within a virtual network. - - :param virtual_network: The virtual network name to list subnets within. - - :param resource_group: The resource group name assigned to the - virtual network. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.subnets_list testnet testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - subnets = __utils__["azurearm.paged_object_to_list"]( - netconn.subnets.list( - resource_group_name=resource_group, virtual_network_name=virtual_network - ) - ) - - for subnet in subnets: - result[subnet["name"]] = subnet - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def subnet_get(name, virtual_network, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a specific subnet. - - :param name: The name of the subnet to query. - - :param virtual_network: The virtual network name containing the - subnet. - - :param resource_group: The resource group name assigned to the - virtual network. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.subnet_get testsubnet testnet testgroup - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - subnet = netconn.subnets.get( - resource_group_name=resource_group, - virtual_network_name=virtual_network, - subnet_name=name, - ) - - result = subnet.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def subnet_create_or_update( - name, address_prefix, virtual_network, resource_group, **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Create or update a subnet. - - :param name: The name assigned to the subnet being created or updated. - - :param address_prefix: A valid CIDR block within the virtual network. - - :param virtual_network: The virtual network name containing the - subnet. - - :param resource_group: The resource group name assigned to the - virtual network. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.subnet_create_or_update testsubnet \ - '10.0.0.0/24' testnet testgroup - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - # Use NSG name to link to the ID of an existing NSG. - if kwargs.get("network_security_group"): - nsg = network_security_group_get( - name=kwargs["network_security_group"], - resource_group=resource_group, - **kwargs - ) - if "error" not in nsg: - kwargs["network_security_group"] = {"id": str(nsg["id"])} - - # Use Route Table name to link to the ID of an existing Route Table. - if kwargs.get("route_table"): - rt_table = route_table_get( - name=kwargs["route_table"], resource_group=resource_group, **kwargs - ) - if "error" not in rt_table: - kwargs["route_table"] = {"id": str(rt_table["id"])} - - try: - snetmodel = __utils__["azurearm.create_object_model"]( - "network", - "Subnet", - address_prefix=address_prefix, - resource_group=resource_group, - **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - subnet = netconn.subnets.create_or_update( - resource_group_name=resource_group, - virtual_network_name=virtual_network, - subnet_name=name, - subnet_parameters=snetmodel, - ) - subnet.wait() - sn_result = subnet.result() - result = sn_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def subnet_delete(name, virtual_network, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a subnet. - - :param name: The name of the subnet to delete. - - :param virtual_network: The virtual network name containing the - subnet. - - :param resource_group: The resource group name assigned to the - virtual network. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.subnet_delete testsubnet testnet testgroup - - """ - result = False - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - subnet = netconn.subnets.delete( - resource_group_name=resource_group, - virtual_network_name=virtual_network, - subnet_name=name, - ) - subnet.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - - return result - - -@_deprecation_message -def virtual_networks_list_all(**kwargs): - """ - .. versionadded:: 2019.2.0 - - List all virtual networks within a subscription. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.virtual_networks_list_all - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - vnets = __utils__["azurearm.paged_object_to_list"]( - netconn.virtual_networks.list_all() - ) - - for vnet in vnets: - result[vnet["name"]] = vnet - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def virtual_networks_list(resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all virtual networks within a resource group. - - :param resource_group: The resource group name to list virtual networks - within. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.virtual_networks_list testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - vnets = __utils__["azurearm.paged_object_to_list"]( - netconn.virtual_networks.list(resource_group_name=resource_group) - ) - - for vnet in vnets: - result[vnet["name"]] = vnet - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -# pylint: disable=invalid-name -@_deprecation_message -def virtual_network_create_or_update(name, address_prefixes, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Create or update a virtual network. - - :param name: The name assigned to the virtual network being - created or updated. - - :param address_prefixes: A list of CIDR blocks which can be used - by subnets within the virtual network. - - :param resource_group: The resource group name assigned to the - virtual network. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.virtual_network_create_or_update \ - testnet ['10.0.0.0/16'] testgroup - - """ - if "location" not in kwargs: - rg_props = __salt__["azurearm_resource.resource_group_get"]( - resource_group, **kwargs - ) - - if "error" in rg_props: - log.error("Unable to determine location from resource group specified.") - return False - kwargs["location"] = rg_props["location"] - - if not isinstance(address_prefixes, list): - log.error("Address prefixes must be specified as a list!") - return False - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - address_space = {"address_prefixes": address_prefixes} - dhcp_options = {"dns_servers": kwargs.get("dns_servers")} - - try: - vnetmodel = __utils__["azurearm.create_object_model"]( - "network", - "VirtualNetwork", - address_space=address_space, - dhcp_options=dhcp_options, - **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - vnet = netconn.virtual_networks.create_or_update( - virtual_network_name=name, - resource_group_name=resource_group, - parameters=vnetmodel, - ) - vnet.wait() - vnet_result = vnet.result() - result = vnet_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def virtual_network_delete(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a virtual network. - - :param name: The name of the virtual network to delete. - - :param resource_group: The resource group name assigned to the - virtual network - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.virtual_network_delete testnet testgroup - - """ - result = False - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - vnet = netconn.virtual_networks.delete( - virtual_network_name=name, resource_group_name=resource_group - ) - vnet.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - - return result - - -@_deprecation_message -def virtual_network_get(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a specific virtual network. - - :param name: The name of the virtual network to query. - - :param resource_group: The resource group name assigned to the - virtual network. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.virtual_network_get testnet testgroup - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - vnet = netconn.virtual_networks.get( - virtual_network_name=name, resource_group_name=resource_group - ) - result = vnet.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def load_balancers_list_all(**kwargs): - """ - .. versionadded:: 2019.2.0 - - List all load balancers within a subscription. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.load_balancers_list_all - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - load_balancers = __utils__["azurearm.paged_object_to_list"]( - netconn.load_balancers.list_all() - ) - - for load_balancer in load_balancers: - result[load_balancer["name"]] = load_balancer - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def load_balancers_list(resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all load balancers within a resource group. - - :param resource_group: The resource group name to list load balancers - within. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.load_balancers_list testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - load_balancers = __utils__["azurearm.paged_object_to_list"]( - netconn.load_balancers.list(resource_group_name=resource_group) - ) - - for load_balancer in load_balancers: - result[load_balancer["name"]] = load_balancer - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def load_balancer_get(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a specific load balancer. - - :param name: The name of the load balancer to query. - - :param resource_group: The resource group name assigned to the - load balancer. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.load_balancer_get testlb testgroup - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - load_balancer = netconn.load_balancers.get( - load_balancer_name=name, resource_group_name=resource_group - ) - result = load_balancer.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def load_balancer_create_or_update(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Create or update a load balancer within a specified resource group. - - :param name: The name of the load balancer to create. - - :param resource_group: The resource group name assigned to the - load balancer. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.load_balancer_create_or_update testlb testgroup - - """ - if "location" not in kwargs: - rg_props = __salt__["azurearm_resource.resource_group_get"]( - resource_group, **kwargs - ) - - if "error" in rg_props: - log.error("Unable to determine location from resource group specified.") - return False - kwargs["location"] = rg_props["location"] - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - if isinstance(kwargs.get("frontend_ip_configurations"), list): - for idx in range(0, len(kwargs["frontend_ip_configurations"])): - # Use Public IP Address name to link to the ID of an existing Public IP - if "public_ip_address" in kwargs["frontend_ip_configurations"][idx]: - pub_ip = public_ip_address_get( - name=kwargs["frontend_ip_configurations"][idx]["public_ip_address"], - resource_group=resource_group, - **kwargs - ) - if "error" not in pub_ip: - kwargs["frontend_ip_configurations"][idx]["public_ip_address"] = { - "id": str(pub_ip["id"]) - } - # Use Subnet name to link to the ID of an existing Subnet - elif "subnet" in kwargs["frontend_ip_configurations"][idx]: - vnets = virtual_networks_list(resource_group=resource_group, **kwargs) - if "error" not in vnets: - for vnet in vnets: - subnets = subnets_list( - virtual_network=vnet, - resource_group=resource_group, - **kwargs - ) - if ( - kwargs["frontend_ip_configurations"][idx]["subnet"] - in subnets - ): - kwargs["frontend_ip_configurations"][idx]["subnet"] = { - "id": str( - subnets[ - kwargs["frontend_ip_configurations"][idx][ - "subnet" - ] - ]["id"] - ) - } - break - - id_url = "/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Network/loadBalancers/{2}/{3}/{4}" - - if isinstance(kwargs.get("load_balancing_rules"), list): - for idx in range(0, len(kwargs["load_balancing_rules"])): - # Link to sub-objects which might be created at the same time as the load balancer - if "frontend_ip_configuration" in kwargs["load_balancing_rules"][idx]: - kwargs["load_balancing_rules"][idx]["frontend_ip_configuration"] = { - "id": id_url.format( - kwargs.get("subscription_id"), - resource_group, - name, - "frontendIPConfigurations", - kwargs["load_balancing_rules"][idx][ - "frontend_ip_configuration" - ], - ) - } - if "backend_address_pool" in kwargs["load_balancing_rules"][idx]: - kwargs["load_balancing_rules"][idx]["backend_address_pool"] = { - "id": id_url.format( - kwargs.get("subscription_id"), - resource_group, - name, - "backendAddressPools", - kwargs["load_balancing_rules"][idx]["backend_address_pool"], - ) - } - if "probe" in kwargs["load_balancing_rules"][idx]: - kwargs["load_balancing_rules"][idx]["probe"] = { - "id": id_url.format( - kwargs.get("subscription_id"), - resource_group, - name, - "probes", - kwargs["load_balancing_rules"][idx]["probe"], - ) - } - - if isinstance(kwargs.get("inbound_nat_rules"), list): - for idx in range(0, len(kwargs["inbound_nat_rules"])): - # Link to sub-objects which might be created at the same time as the load balancer - if "frontend_ip_configuration" in kwargs["inbound_nat_rules"][idx]: - kwargs["inbound_nat_rules"][idx]["frontend_ip_configuration"] = { - "id": id_url.format( - kwargs.get("subscription_id"), - resource_group, - name, - "frontendIPConfigurations", - kwargs["inbound_nat_rules"][idx]["frontend_ip_configuration"], - ) - } - - if isinstance(kwargs.get("inbound_nat_pools"), list): - for idx in range(0, len(kwargs["inbound_nat_pools"])): - # Link to sub-objects which might be created at the same time as the load balancer - if "frontend_ip_configuration" in kwargs["inbound_nat_pools"][idx]: - kwargs["inbound_nat_pools"][idx]["frontend_ip_configuration"] = { - "id": id_url.format( - kwargs.get("subscription_id"), - resource_group, - name, - "frontendIPConfigurations", - kwargs["inbound_nat_pools"][idx]["frontend_ip_configuration"], - ) - } - - if isinstance(kwargs.get("outbound_nat_rules"), list): - for idx in range(0, len(kwargs["outbound_nat_rules"])): - # Link to sub-objects which might be created at the same time as the load balancer - if "frontend_ip_configuration" in kwargs["outbound_nat_rules"][idx]: - kwargs["outbound_nat_rules"][idx]["frontend_ip_configuration"] = { - "id": id_url.format( - kwargs.get("subscription_id"), - resource_group, - name, - "frontendIPConfigurations", - kwargs["outbound_nat_rules"][idx]["frontend_ip_configuration"], - ) - } - if "backend_address_pool" in kwargs["outbound_nat_rules"][idx]: - kwargs["outbound_nat_rules"][idx]["backend_address_pool"] = { - "id": id_url.format( - kwargs.get("subscription_id"), - resource_group, - name, - "backendAddressPools", - kwargs["outbound_nat_rules"][idx]["backend_address_pool"], - ) - } - - try: - lbmodel = __utils__["azurearm.create_object_model"]( - "network", "LoadBalancer", **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - load_balancer = netconn.load_balancers.create_or_update( - resource_group_name=resource_group, - load_balancer_name=name, - parameters=lbmodel, - ) - load_balancer.wait() - lb_result = load_balancer.result() - result = lb_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def load_balancer_delete(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a load balancer. - - :param name: The name of the load balancer to delete. - - :param resource_group: The resource group name assigned to the - load balancer. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.load_balancer_delete testlb testgroup - - """ - result = False - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - load_balancer = netconn.load_balancers.delete( - load_balancer_name=name, resource_group_name=resource_group - ) - load_balancer.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - - return result - - -@_deprecation_message -def usages_list(location, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List subscription network usage for a location. - - :param location: The Azure location to query for network usage. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.usages_list westus - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - result = __utils__["azurearm.paged_object_to_list"]( - netconn.usages.list(location) - ) - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def network_interface_delete(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a network interface. - - :param name: The name of the network interface to delete. - - :param resource_group: The resource group name assigned to the - network interface. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.network_interface_delete test-iface0 testgroup - - """ - result = False - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - nic = netconn.network_interfaces.delete( - network_interface_name=name, resource_group_name=resource_group - ) - nic.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - - return result - - -@_deprecation_message -def network_interface_get(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a specific network interface. - - :param name: The name of the network interface to query. - - :param resource_group: The resource group name assigned to the - network interface. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.network_interface_get test-iface0 testgroup - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - nic = netconn.network_interfaces.get( - network_interface_name=name, resource_group_name=resource_group - ) - result = nic.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -# pylint: disable=invalid-name -@_deprecation_message -def network_interface_create_or_update( - name, ip_configurations, subnet, virtual_network, resource_group, **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Create or update a network interface within a specified resource group. - - :param name: The name of the network interface to create. - - :param ip_configurations: A list of dictionaries representing valid - NetworkInterfaceIPConfiguration objects. The 'name' key is required at - minimum. At least one IP Configuration must be present. - - :param subnet: The name of the subnet assigned to the network interface. - - :param virtual_network: The name of the virtual network assigned to the subnet. - - :param resource_group: The resource group name assigned to the - virtual network. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.network_interface_create_or_update test-iface0 [{'name': 'testipconfig1'}] \ - testsubnet testnet testgroup - - """ - if "location" not in kwargs: - rg_props = __salt__["azurearm_resource.resource_group_get"]( - resource_group, **kwargs - ) - - if "error" in rg_props: - log.error("Unable to determine location from resource group specified.") - return False - kwargs["location"] = rg_props["location"] - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - # Use NSG name to link to the ID of an existing NSG. - if kwargs.get("network_security_group"): - nsg = network_security_group_get( - name=kwargs["network_security_group"], - resource_group=resource_group, - **kwargs - ) - if "error" not in nsg: - kwargs["network_security_group"] = {"id": str(nsg["id"])} - - # Use VM name to link to the ID of an existing VM. - if kwargs.get("virtual_machine"): - vm_instance = __salt__["azurearm_compute.virtual_machine_get"]( - name=kwargs["virtual_machine"], resource_group=resource_group, **kwargs - ) - if "error" not in vm_instance: - kwargs["virtual_machine"] = {"id": str(vm_instance["id"])} - - # Loop through IP Configurations and build each dictionary to pass to model creation. - if isinstance(ip_configurations, list): - subnet = subnet_get( - name=subnet, - virtual_network=virtual_network, - resource_group=resource_group, - **kwargs - ) - if "error" not in subnet: - subnet = {"id": str(subnet["id"])} - for ipconfig in ip_configurations: - if "name" in ipconfig: - ipconfig["subnet"] = subnet - if isinstance( - ipconfig.get("application_gateway_backend_address_pools"), list - ): - # TODO: Add ID lookup for referenced object names - pass - if isinstance( - ipconfig.get("load_balancer_backend_address_pools"), list - ): - # TODO: Add ID lookup for referenced object names - pass - if isinstance( - ipconfig.get("load_balancer_inbound_nat_rules"), list - ): - # TODO: Add ID lookup for referenced object names - pass - if ipconfig.get("public_ip_address"): - pub_ip = public_ip_address_get( - name=ipconfig["public_ip_address"], - resource_group=resource_group, - **kwargs - ) - if "error" not in pub_ip: - ipconfig["public_ip_address"] = {"id": str(pub_ip["id"])} - - try: - nicmodel = __utils__["azurearm.create_object_model"]( - "network", "NetworkInterface", ip_configurations=ip_configurations, **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - interface = netconn.network_interfaces.create_or_update( - resource_group_name=resource_group, - network_interface_name=name, - parameters=nicmodel, - ) - interface.wait() - nic_result = interface.result() - result = nic_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def network_interfaces_list_all(**kwargs): - """ - .. versionadded:: 2019.2.0 - - List all network interfaces within a subscription. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.network_interfaces_list_all - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - nics = __utils__["azurearm.paged_object_to_list"]( - netconn.network_interfaces.list_all() - ) - - for nic in nics: - result[nic["name"]] = nic - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def network_interfaces_list(resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all network interfaces within a resource group. - - :param resource_group: The resource group name to list network - interfaces within. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.network_interfaces_list testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - nics = __utils__["azurearm.paged_object_to_list"]( - netconn.network_interfaces.list(resource_group_name=resource_group) - ) - - for nic in nics: - result[nic["name"]] = nic - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -# pylint: disable=invalid-name -@_deprecation_message -def network_interface_get_effective_route_table(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get all route tables for a specific network interface. - - :param name: The name of the network interface to query. - - :param resource_group: The resource group name assigned to the - network interface. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.network_interface_get_effective_route_table test-iface0 testgroup - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - nic = netconn.network_interfaces.get_effective_route_table( - network_interface_name=name, resource_group_name=resource_group - ) - nic.wait() - tables = nic.result() - tables = tables.as_dict() - result = tables["value"] - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -# pylint: disable=invalid-name -@_deprecation_message -def network_interface_list_effective_network_security_groups( - name, resource_group, **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Get all network security groups applied to a specific network interface. - - :param name: The name of the network interface to query. - - :param resource_group: The resource group name assigned to the - network interface. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.network_interface_list_effective_network_security_groups test-iface0 testgroup - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - nic = netconn.network_interfaces.list_effective_network_security_groups( - network_interface_name=name, resource_group_name=resource_group - ) - nic.wait() - groups = nic.result() - groups = groups.as_dict() - result = groups["value"] - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -# pylint: disable=invalid-name -@_deprecation_message -def list_virtual_machine_scale_set_vm_network_interfaces( - scale_set, vm_index, resource_group, **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Get information about all network interfaces in a specific virtual machine within a scale set. - - :param scale_set: The name of the scale set to query. - - :param vm_index: The virtual machine index. - - :param resource_group: The resource group name assigned to the - scale set. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.list_virtual_machine_scale_set_vm_network_interfaces testset testvm testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - nics = __utils__["azurearm.paged_object_to_list"]( - netconn.network_interfaces.list_virtual_machine_scale_set_vm_network_interfaces( - virtual_machine_scale_set_name=scale_set, - virtualmachine_index=vm_index, - resource_group_name=resource_group, - ) - ) - - for nic in nics: - result[nic["name"]] = nic - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -# pylint: disable=invalid-name -@_deprecation_message -def list_virtual_machine_scale_set_network_interfaces( - scale_set, resource_group, **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Get information about all network interfaces within a scale set. - - :param scale_set: The name of the scale set to query. - - :param resource_group: The resource group name assigned to the - scale set. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.list_virtual_machine_scale_set_vm_network_interfaces testset testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - nics = __utils__["azurearm.paged_object_to_list"]( - netconn.network_interfaces.list_virtual_machine_scale_set_network_interfaces( - virtual_machine_scale_set_name=scale_set, - resource_group_name=resource_group, - ) - ) - - for nic in nics: - result[nic["name"]] = nic - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -# pylint: disable=invalid-name -@_deprecation_message -def get_virtual_machine_scale_set_network_interface( - name, scale_set, vm_index, resource_group, **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Get information about a specific network interface within a scale set. - - :param name: The name of the network interface to query. - - :param scale_set: The name of the scale set containing the interface. - - :param vm_index: The virtual machine index. - - :param resource_group: The resource group name assigned to the - scale set. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.get_virtual_machine_scale_set_network_interface test-iface0 testset testvm testgroup - - """ - expand = kwargs.get("expand") - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - nic = netconn.network_interfaces.list_virtual_machine_scale_set_vm_network_interfaces( - network_interface_name=name, - virtual_machine_scale_set_name=scale_set, - virtualmachine_index=vm_index, - resource_group_name=resource_group, - exapnd=expand, - ) - - result = nic.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def public_ip_address_delete(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a public IP address. - - :param name: The name of the public IP address to delete. - - :param resource_group: The resource group name assigned to the - public IP address. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.public_ip_address_delete test-pub-ip testgroup - - """ - result = False - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - pub_ip = netconn.public_ip_addresses.delete( - public_ip_address_name=name, resource_group_name=resource_group - ) - pub_ip.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - - return result - - -@_deprecation_message -def public_ip_address_get(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a specific public IP address. - - :param name: The name of the public IP address to query. - - :param resource_group: The resource group name assigned to the - public IP address. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.public_ip_address_get test-pub-ip testgroup - - """ - expand = kwargs.get("expand") - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - try: - pub_ip = netconn.public_ip_addresses.get( - public_ip_address_name=name, - resource_group_name=resource_group, - expand=expand, - ) - result = pub_ip.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def public_ip_address_create_or_update(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Create or update a public IP address within a specified resource group. - - :param name: The name of the public IP address to create. - - :param resource_group: The resource group name assigned to the - public IP address. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.public_ip_address_create_or_update test-ip-0 testgroup - - """ - if "location" not in kwargs: - rg_props = __salt__["azurearm_resource.resource_group_get"]( - resource_group, **kwargs - ) - - if "error" in rg_props: - log.error("Unable to determine location from resource group specified.") - return False - kwargs["location"] = rg_props["location"] - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - try: - pub_ip_model = __utils__["azurearm.create_object_model"]( - "network", "PublicIPAddress", **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - ip = netconn.public_ip_addresses.create_or_update( - resource_group_name=resource_group, - public_ip_address_name=name, - parameters=pub_ip_model, - ) - ip.wait() - ip_result = ip.result() - result = ip_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def public_ip_addresses_list_all(**kwargs): - """ - .. versionadded:: 2019.2.0 - - List all public IP addresses within a subscription. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.public_ip_addresses_list_all - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - pub_ips = __utils__["azurearm.paged_object_to_list"]( - netconn.public_ip_addresses.list_all() - ) - - for ip in pub_ips: - result[ip["name"]] = ip - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def public_ip_addresses_list(resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all public IP addresses within a resource group. - - :param resource_group: The resource group name to list public IP - addresses within. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.public_ip_addresses_list testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - pub_ips = __utils__["azurearm.paged_object_to_list"]( - netconn.public_ip_addresses.list(resource_group_name=resource_group) - ) - - for ip in pub_ips: - result[ip["name"]] = ip - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def route_filter_rule_delete(name, route_filter, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a route filter rule. - - :param name: The route filter rule to delete. - - :param route_filter: The route filter containing the rule. - - :param resource_group: The resource group name assigned to the - route filter. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_filter_rule_delete test-rule test-filter testgroup - - """ - result = False - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - rule = netconn.route_filter_rules.delete( - resource_group_name=resource_group, - route_filter_name=route_filter, - rule_name=name, - ) - rule.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - - return result - - -@_deprecation_message -def route_filter_rule_get(name, route_filter, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a specific route filter rule. - - :param name: The route filter rule to query. - - :param route_filter: The route filter containing the rule. - - :param resource_group: The resource group name assigned to the - route filter. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_filter_rule_get test-rule test-filter testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - rule = netconn.route_filter_rules.get( - resource_group_name=resource_group, - route_filter_name=route_filter, - rule_name=name, - ) - - result = rule.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def route_filter_rule_create_or_update( - name, access, communities, route_filter, resource_group, **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Create or update a rule within a specified route filter. - - :param name: The name of the rule to create. - - :param access: The access type of the rule. Valid values are 'Allow' and 'Deny'. - - :param communities: A list of BGP communities to filter on. - - :param route_filter: The name of the route filter containing the rule. - - :param resource_group: The resource group name assigned to the - route filter. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_filter_rule_create_or_update \ - test-rule allow "['12076:51006']" test-filter testgroup - - """ - if not isinstance(communities, list): - log.error("The communities parameter must be a list of strings!") - return False - - if "location" not in kwargs: - rg_props = __salt__["azurearm_resource.resource_group_get"]( - resource_group, **kwargs - ) - - if "error" in rg_props: - log.error("Unable to determine location from resource group specified.") - return False - kwargs["location"] = rg_props["location"] - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - try: - rule_model = __utils__["azurearm.create_object_model"]( - "network", - "RouteFilterRule", - access=access, - communities=communities, - **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - rule = netconn.route_filter_rules.create_or_update( - resource_group_name=resource_group, - route_filter_name=route_filter, - rule_name=name, - route_filter_rule_parameters=rule_model, - ) - rule.wait() - rule_result = rule.result() - result = rule_result.as_dict() - except CloudError as exc: - message = str(exc) - if kwargs.get("subscription_id") == str(message).strip(): - message = "Subscription not authorized for this operation!" - __utils__["azurearm.log_cloud_error"]("network", message, **kwargs) - result = {"error": message} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def route_filter_rules_list(route_filter, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all routes within a route filter. - - :param route_filter: The route filter to query. - - :param resource_group: The resource group name assigned to the - route filter. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_filter_rules_list test-filter testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - rules = __utils__["azurearm.paged_object_to_list"]( - netconn.route_filter_rules.list_by_route_filter( - resource_group_name=resource_group, route_filter_name=route_filter - ) - ) - - for rule in rules: - result[rule["name"]] = rule - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def route_filter_delete(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a route filter. - - :param name: The name of the route filter to delete. - - :param resource_group: The resource group name assigned to the - route filter. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_filter_delete test-filter testgroup - - """ - result = False - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - route_filter = netconn.route_filters.delete( - route_filter_name=name, resource_group_name=resource_group - ) - route_filter.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - - return result - - -@_deprecation_message -def route_filter_get(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a specific route filter. - - :param name: The name of the route table to query. - - :param resource_group: The resource group name assigned to the - route filter. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_filter_get test-filter testgroup - - """ - expand = kwargs.get("expand") - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - try: - route_filter = netconn.route_filters.get( - route_filter_name=name, resource_group_name=resource_group, expand=expand - ) - result = route_filter.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def route_filter_create_or_update(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Create or update a route filter within a specified resource group. - - :param name: The name of the route filter to create. - - :param resource_group: The resource group name assigned to the - route filter. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_filter_create_or_update test-filter testgroup - - """ - if "location" not in kwargs: - rg_props = __salt__["azurearm_resource.resource_group_get"]( - resource_group, **kwargs - ) - - if "error" in rg_props: - log.error("Unable to determine location from resource group specified.") - return False - kwargs["location"] = rg_props["location"] - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - try: - rt_filter_model = __utils__["azurearm.create_object_model"]( - "network", "RouteFilter", **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - rt_filter = netconn.route_filters.create_or_update( - resource_group_name=resource_group, - route_filter_name=name, - route_filter_parameters=rt_filter_model, - ) - rt_filter.wait() - rt_result = rt_filter.result() - result = rt_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def route_filters_list(resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all route filters within a resource group. - - :param resource_group: The resource group name to list route - filters within. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_filters_list testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - filters = __utils__["azurearm.paged_object_to_list"]( - netconn.route_filters.list_by_resource_group( - resource_group_name=resource_group - ) - ) - - for route_filter in filters: - result[route_filter["name"]] = route_filter - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def route_filters_list_all(**kwargs): - """ - .. versionadded:: 2019.2.0 - - List all route filters within a subscription. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_filters_list_all - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - filters = __utils__["azurearm.paged_object_to_list"]( - netconn.route_filters.list() - ) - - for route_filter in filters: - result[route_filter["name"]] = route_filter - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def route_delete(name, route_table, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a route from a route table. - - :param name: The route to delete. - - :param route_table: The route table containing the route. - - :param resource_group: The resource group name assigned to the - route table. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_delete test-rt test-rt-table testgroup - - """ - result = False - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - route = netconn.routes.delete( - resource_group_name=resource_group, - route_table_name=route_table, - route_name=name, - ) - route.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - - return result - - -@_deprecation_message -def route_get(name, route_table, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a specific route. - - :param name: The route to query. - - :param route_table: The route table containing the route. - - :param resource_group: The resource group name assigned to the - route table. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_get test-rt test-rt-table testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - route = netconn.routes.get( - resource_group_name=resource_group, - route_table_name=route_table, - route_name=name, - ) - - result = route.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def route_create_or_update( - name, - address_prefix, - next_hop_type, - route_table, - resource_group, - next_hop_ip_address=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Create or update a route within a specified route table. - - :param name: The name of the route to create. - - :param address_prefix: The destination CIDR to which the route applies. - - :param next_hop_type: The type of Azure hop the packet should be sent to. Possible values are: - 'VirtualNetworkGateway', 'VnetLocal', 'Internet', 'VirtualAppliance', and 'None'. - - :param next_hop_ip_address: Optional IP address to which packets should be forwarded. Next hop - values are only allowed in routes where the next_hop_type is 'VirtualAppliance'. - - :param route_table: The name of the route table containing the route. - - :param resource_group: The resource group name assigned to the - route table. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_create_or_update test-rt '10.0.0.0/8' test-rt-table testgroup - - """ - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - try: - rt_model = __utils__["azurearm.create_object_model"]( - "network", - "Route", - address_prefix=address_prefix, - next_hop_type=next_hop_type, - next_hop_ip_address=next_hop_ip_address, - **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - route = netconn.routes.create_or_update( - resource_group_name=resource_group, - route_table_name=route_table, - route_name=name, - route_parameters=rt_model, - ) - route.wait() - rt_result = route.result() - result = rt_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def routes_list(route_table, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all routes within a route table. - - :param route_table: The route table to query. - - :param resource_group: The resource group name assigned to the - route table. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.routes_list test-rt-table testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - routes = __utils__["azurearm.paged_object_to_list"]( - netconn.routes.list( - resource_group_name=resource_group, route_table_name=route_table - ) - ) - - for route in routes: - result[route["name"]] = route - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def route_table_delete(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a route table. - - :param name: The name of the route table to delete. - - :param resource_group: The resource group name assigned to the - route table. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_table_delete test-rt-table testgroup - - """ - result = False - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - table = netconn.route_tables.delete( - route_table_name=name, resource_group_name=resource_group - ) - table.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - - return result - - -@_deprecation_message -def route_table_get(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a specific route table. - - :param name: The name of the route table to query. - - :param resource_group: The resource group name assigned to the - route table. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_table_get test-rt-table testgroup - - """ - expand = kwargs.get("expand") - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - try: - table = netconn.route_tables.get( - route_table_name=name, resource_group_name=resource_group, expand=expand - ) - result = table.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def route_table_create_or_update(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Create or update a route table within a specified resource group. - - :param name: The name of the route table to create. - - :param resource_group: The resource group name assigned to the - route table. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_table_create_or_update test-rt-table testgroup - - """ - if "location" not in kwargs: - rg_props = __salt__["azurearm_resource.resource_group_get"]( - resource_group, **kwargs - ) - - if "error" in rg_props: - log.error("Unable to determine location from resource group specified.") - return False - kwargs["location"] = rg_props["location"] - - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - - try: - rt_tbl_model = __utils__["azurearm.create_object_model"]( - "network", "RouteTable", **kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - table = netconn.route_tables.create_or_update( - resource_group_name=resource_group, - route_table_name=name, - parameters=rt_tbl_model, - ) - table.wait() - tbl_result = table.result() - result = tbl_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def route_tables_list(resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all route tables within a resource group. - - :param resource_group: The resource group name to list route - tables within. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_tables_list testgroup - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - tables = __utils__["azurearm.paged_object_to_list"]( - netconn.route_tables.list(resource_group_name=resource_group) - ) - - for table in tables: - result[table["name"]] = table - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def route_tables_list_all(**kwargs): - """ - .. versionadded:: 2019.2.0 - - List all route tables within a subscription. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_network.route_tables_list_all - - """ - result = {} - netconn = __utils__["azurearm.get_client"]("network", **kwargs) - try: - tables = __utils__["azurearm.paged_object_to_list"]( - netconn.route_tables.list_all() - ) - - for table in tables: - result[table["name"]] = table - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("network", str(exc), **kwargs) - result = {"error": str(exc)} - - return result diff --git a/salt/modules/azurearm_resource.py b/salt/modules/azurearm_resource.py deleted file mode 100644 index b1b865a81d1..00000000000 --- a/salt/modules/azurearm_resource.py +++ /dev/null @@ -1,1253 +0,0 @@ -""" -Azure (ARM) Resource Execution Module - -.. versionadded:: 2019.2.0 - -.. warning:: - - This cloud provider will be removed from Salt in version 3007 in favor of - the `saltext.azurerm Salt Extension - `_ - -:maintainer: -:maturity: new -:depends: - * `azure `_ >= 2.0.0 - * `azure-common `_ >= 1.1.8 - * `azure-mgmt `_ >= 1.0.0 - * `azure-mgmt-compute `_ >= 1.0.0 - * `azure-mgmt-network `_ >= 1.7.1 - * `azure-mgmt-resource `_ >= 1.1.0 - * `azure-mgmt-storage `_ >= 1.0.0 - * `azure-mgmt-web `_ >= 0.32.0 - * `azure-storage `_ >= 0.34.3 - * `msrestazure `_ >= 0.4.21 -:platform: linux - -:configuration: This module requires Azure Resource Manager credentials to be passed as keyword arguments - to every function in order to work properly. - - Required provider parameters: - - if using username and password: - * ``subscription_id`` - * ``username`` - * ``password`` - - if using a service principal: - * ``subscription_id`` - * ``tenant`` - * ``client_id`` - * ``secret`` - - Optional provider parameters: - -**cloud_environment**: Used to point the cloud driver to different API endpoints, such as Azure GovCloud. - Possible values: - * ``AZURE_PUBLIC_CLOUD`` (default) - * ``AZURE_CHINA_CLOUD`` - * ``AZURE_US_GOV_CLOUD`` - * ``AZURE_GERMAN_CLOUD`` - -""" - -# Python libs - -import logging -from functools import wraps - -# Salt Libs -import salt.utils.azurearm -import salt.utils.json - -# Azure libs -HAS_LIBS = False -try: - import azure.mgmt.resource.resources.models # pylint: disable=unused-import - from msrest.exceptions import SerializationError - from msrestazure.azure_exceptions import CloudError - - HAS_LIBS = True -except ImportError: - pass - -__virtualname__ = "azurearm_resource" - -log = logging.getLogger(__name__) - - -def __virtual__(): - if not HAS_LIBS: - return ( - False, - "The following dependencies are required to use the AzureARM modules: " - "Microsoft Azure SDK for Python >= 2.0rc6, " - "MS REST Azure (msrestazure) >= 0.4", - ) - - return __virtualname__ - - -def _deprecation_message(function): - """ - Decorator wrapper to warn about azurearm deprecation - """ - - @wraps(function) - def wrapped(*args, **kwargs): - salt.utils.versions.warn_until( - "Chlorine", - "The 'azurearm' functionality in Salt has been deprecated and its " - "functionality will be removed in version 3007 in favor of the " - "saltext.azurerm Salt Extension. " - "(https://github.com/salt-extensions/saltext-azurerm)", - category=FutureWarning, - ) - ret = function(*args, **salt.utils.args.clean_kwargs(**kwargs)) - return ret - - return wrapped - - -@_deprecation_message -def resource_groups_list(**kwargs): - """ - .. versionadded:: 2019.2.0 - - List all resource groups within a subscription. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.resource_groups_list - - """ - result = {} - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - try: - groups = __utils__["azurearm.paged_object_to_list"]( - resconn.resource_groups.list() - ) - - for group in groups: - result[group["name"]] = group - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def resource_group_check_existence(name, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Check for the existence of a named resource group in the current subscription. - - :param name: The resource group name to check. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.resource_group_check_existence testgroup - - """ - result = False - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - try: - result = resconn.resource_groups.check_existence(name) - - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - - return result - - -@_deprecation_message -def resource_group_get(name, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get a dictionary representing a resource group's properties. - - :param name: The resource group name to get. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.resource_group_get testgroup - - """ - result = {} - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - try: - group = resconn.resource_groups.get(name) - result = group.as_dict() - - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def resource_group_create_or_update( - name, location, **kwargs -): # pylint: disable=invalid-name - """ - .. versionadded:: 2019.2.0 - - Create or update a resource group in a given location. - - :param name: The name of the resource group to create or update. - - :param location: The location of the resource group. This value - is not able to be updated once the resource group is created. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.resource_group_create_or_update testgroup westus - - """ - result = {} - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - resource_group_params = { - "location": location, - "managed_by": kwargs.get("managed_by"), - "tags": kwargs.get("tags"), - } - try: - group = resconn.resource_groups.create_or_update(name, resource_group_params) - result = group.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def resource_group_delete(name, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a resource group from the subscription. - - :param name: The resource group name to delete. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.resource_group_delete testgroup - - """ - result = False - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - try: - group = resconn.resource_groups.delete(name) - group.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - - return result - - -@_deprecation_message -def deployment_operation_get(operation, deployment, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get a deployment operation within a deployment. - - :param operation: The operation ID of the operation within the deployment. - - :param deployment: The name of the deployment containing the operation. - - :param resource_group: The resource group name assigned to the - deployment. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.deployment_operation_get XXXXX testdeploy testgroup - - """ - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - try: - operation = resconn.deployment_operations.get( - resource_group_name=resource_group, - deployment_name=deployment, - operation_id=operation, - ) - - result = operation.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def deployment_operations_list(name, resource_group, result_limit=10, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all deployment operations within a deployment. - - :param name: The name of the deployment to query. - - :param resource_group: The resource group name assigned to the - deployment. - - :param result_limit: (Default: 10) The limit on the list of deployment - operations. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.deployment_operations_list testdeploy testgroup - - """ - result = {} - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - try: - operations = __utils__["azurearm.paged_object_to_list"]( - resconn.deployment_operations.list( - resource_group_name=resource_group, - deployment_name=name, - top=result_limit, - ) - ) - - for oper in operations: - result[oper["operation_id"]] = oper - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def deployment_delete(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a deployment. - - :param name: The name of the deployment to delete. - - :param resource_group: The resource group name assigned to the - deployment. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.deployment_delete testdeploy testgroup - - """ - result = False - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - try: - deploy = resconn.deployments.delete( - deployment_name=name, resource_group_name=resource_group - ) - deploy.wait() - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - - return result - - -@_deprecation_message -def deployment_check_existence(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Check the existence of a deployment. - - :param name: The name of the deployment to query. - - :param resource_group: The resource group name assigned to the - deployment. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.deployment_check_existence testdeploy testgroup - - """ - result = False - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - try: - result = resconn.deployments.check_existence( - deployment_name=name, resource_group_name=resource_group - ) - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - - return result - - -@_deprecation_message -def deployment_create_or_update( - name, - resource_group, - deploy_mode="incremental", - debug_setting="none", - deploy_params=None, - parameters_link=None, - deploy_template=None, - template_link=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Deploys resources to a resource group. - - :param name: The name of the deployment to create or update. - - :param resource_group: The resource group name assigned to the - deployment. - - :param deploy_mode: The mode that is used to deploy resources. This value can be either - 'incremental' or 'complete'. In Incremental mode, resources are deployed without deleting - existing resources that are not included in the template. In Complete mode, resources - are deployed and existing resources in the resource group that are not included in - the template are deleted. Be careful when using Complete mode as you may - unintentionally delete resources. - - :param debug_setting: The debug setting of the deployment. The permitted values are 'none', - 'requestContent', 'responseContent', or 'requestContent,responseContent'. By logging - information about the request or response, you could potentially expose sensitive data - that is retrieved through the deployment operations. - - :param deploy_params: JSON string containing name and value pairs that define the deployment - parameters for the template. You use this element when you want to provide the parameter - values directly in the request rather than link to an existing parameter file. Use either - the parameters_link property or the deploy_params property, but not both. - - :param parameters_link: The URI of a parameters file. You use this element to link to an existing - parameters file. Use either the parameters_link property or the deploy_params property, but not both. - - :param deploy_template: JSON string of template content. You use this element when you want to pass - the template syntax directly in the request rather than link to an existing template. Use either - the template_link property or the deploy_template property, but not both. - - :param template_link: The URI of the template. Use either the template_link property or the - deploy_template property, but not both. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.deployment_create_or_update testdeploy testgroup - - """ - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - - prop_kwargs = {"mode": deploy_mode} - prop_kwargs["debug_setting"] = {"detail_level": debug_setting} - - if deploy_params: - prop_kwargs["parameters"] = deploy_params - else: - if isinstance(parameters_link, dict): - prop_kwargs["parameters_link"] = parameters_link - else: - prop_kwargs["parameters_link"] = {"uri": parameters_link} - - if deploy_template: - prop_kwargs["template"] = deploy_template - else: - if isinstance(template_link, dict): - prop_kwargs["template_link"] = template_link - else: - prop_kwargs["template_link"] = {"uri": template_link} - - deploy_kwargs = kwargs.copy() - deploy_kwargs.update(prop_kwargs) - - try: - deploy_model = __utils__["azurearm.create_object_model"]( - "resource", "DeploymentProperties", **deploy_kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - validate = deployment_validate( - name=name, resource_group=resource_group, **deploy_kwargs - ) - if "error" in validate: - result = validate - else: - deploy = resconn.deployments.create_or_update( - deployment_name=name, - resource_group_name=resource_group, - properties=deploy_model, - ) - deploy.wait() - deploy_result = deploy.result() - result = deploy_result.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def deployment_get(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a specific deployment. - - :param name: The name of the deployment to query. - - :param resource_group: The resource group name assigned to the - deployment. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.deployment_get testdeploy testgroup - - """ - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - try: - deploy = resconn.deployments.get( - deployment_name=name, resource_group_name=resource_group - ) - result = deploy.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def deployment_cancel(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Cancel a deployment if in 'Accepted' or 'Running' state. - - :param name: The name of the deployment to cancel. - - :param resource_group: The resource group name assigned to the - deployment. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.deployment_cancel testdeploy testgroup - - """ - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - try: - resconn.deployments.cancel( - deployment_name=name, resource_group_name=resource_group - ) - result = {"result": True} - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc), "result": False} - - return result - - -@_deprecation_message -def deployment_validate( - name, - resource_group, - deploy_mode=None, - debug_setting=None, - deploy_params=None, - parameters_link=None, - deploy_template=None, - template_link=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Validates whether the specified template is syntactically correct - and will be accepted by Azure Resource Manager. - - :param name: The name of the deployment to validate. - - :param resource_group: The resource group name assigned to the - deployment. - - :param deploy_mode: The mode that is used to deploy resources. This value can be either - 'incremental' or 'complete'. In Incremental mode, resources are deployed without deleting - existing resources that are not included in the template. In Complete mode, resources - are deployed and existing resources in the resource group that are not included in - the template are deleted. Be careful when using Complete mode as you may - unintentionally delete resources. - - :param debug_setting: The debug setting of the deployment. The permitted values are 'none', - 'requestContent', 'responseContent', or 'requestContent,responseContent'. By logging - information about the request or response, you could potentially expose sensitive data - that is retrieved through the deployment operations. - - :param deploy_params: JSON string containing name and value pairs that define the deployment - parameters for the template. You use this element when you want to provide the parameter - values directly in the request rather than link to an existing parameter file. Use either - the parameters_link property or the deploy_params property, but not both. - - :param parameters_link: The URI of a parameters file. You use this element to link to an existing - parameters file. Use either the parameters_link property or the deploy_params property, but not both. - - :param deploy_template: JSON string of template content. You use this element when you want to pass - the template syntax directly in the request rather than link to an existing template. Use either - the template_link property or the deploy_template property, but not both. - - :param template_link: The URI of the template. Use either the template_link property or the - deploy_template property, but not both. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.deployment_validate testdeploy testgroup - - """ - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - - prop_kwargs = {"mode": deploy_mode} - prop_kwargs["debug_setting"] = {"detail_level": debug_setting} - - if deploy_params: - prop_kwargs["parameters"] = deploy_params - else: - if isinstance(parameters_link, dict): - prop_kwargs["parameters_link"] = parameters_link - else: - prop_kwargs["parameters_link"] = {"uri": parameters_link} - - if deploy_template: - prop_kwargs["template"] = deploy_template - else: - if isinstance(template_link, dict): - prop_kwargs["template_link"] = template_link - else: - prop_kwargs["template_link"] = {"uri": template_link} - - deploy_kwargs = kwargs.copy() - deploy_kwargs.update(prop_kwargs) - - try: - deploy_model = __utils__["azurearm.create_object_model"]( - "resource", "DeploymentProperties", **deploy_kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - local_validation = deploy_model.validate() - if local_validation: - raise local_validation[0] - - deploy = resconn.deployments.validate( - deployment_name=name, - resource_group_name=resource_group, - properties=deploy_model, - ) - result = deploy.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def deployment_export_template(name, resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Exports the template used for the specified deployment. - - :param name: The name of the deployment to query. - - :param resource_group: The resource group name assigned to the - deployment. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.deployment_export_template testdeploy testgroup - - """ - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - try: - deploy = resconn.deployments.export_template( - deployment_name=name, resource_group_name=resource_group - ) - result = deploy.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def deployments_list(resource_group, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all deployments within a resource group. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.deployments_list testgroup - - """ - result = {} - resconn = __utils__["azurearm.get_client"]("resource", **kwargs) - try: - deployments = __utils__["azurearm.paged_object_to_list"]( - resconn.deployments.list_by_resource_group( - resource_group_name=resource_group - ) - ) - - for deploy in deployments: - result[deploy["name"]] = deploy - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def subscriptions_list_locations(subscription_id=None, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all locations for a subscription. - - :param subscription_id: The ID of the subscription to query. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.subscriptions_list_locations XXXXXXXX - - """ - result = {} - - if not subscription_id: - subscription_id = kwargs.get("subscription_id") - elif not kwargs.get("subscription_id"): - kwargs["subscription_id"] = subscription_id - - subconn = __utils__["azurearm.get_client"]("subscription", **kwargs) - try: - locations = __utils__["azurearm.paged_object_to_list"]( - subconn.subscriptions.list_locations( - subscription_id=kwargs["subscription_id"] - ) - ) - - for loc in locations: - result[loc["name"]] = loc - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def subscription_get(subscription_id=None, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a subscription. - - :param subscription_id: The ID of the subscription to query. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.subscription_get XXXXXXXX - - """ - result = {} - - if not subscription_id: - subscription_id = kwargs.get("subscription_id") - elif not kwargs.get("subscription_id"): - kwargs["subscription_id"] = subscription_id - - subconn = __utils__["azurearm.get_client"]("subscription", **kwargs) - try: - subscription = subconn.subscriptions.get( - subscription_id=kwargs.get("subscription_id") - ) - - result = subscription.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def subscriptions_list(**kwargs): - """ - .. versionadded:: 2019.2.0 - - List all subscriptions for a tenant. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.subscriptions_list - - """ - result = {} - subconn = __utils__["azurearm.get_client"]("subscription", **kwargs) - try: - subs = __utils__["azurearm.paged_object_to_list"](subconn.subscriptions.list()) - - for sub in subs: - result[sub["subscription_id"]] = sub - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def tenants_list(**kwargs): - """ - .. versionadded:: 2019.2.0 - - List all tenants for your account. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.tenants_list - - """ - result = {} - subconn = __utils__["azurearm.get_client"]("subscription", **kwargs) - try: - tenants = __utils__["azurearm.paged_object_to_list"](subconn.tenants.list()) - - for tenant in tenants: - result[tenant["tenant_id"]] = tenant - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def policy_assignment_delete(name, scope, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a policy assignment. - - :param name: The name of the policy assignment to delete. - - :param scope: The scope of the policy assignment. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.policy_assignment_delete testassign \ - /subscriptions/bc75htn-a0fhsi-349b-56gh-4fghti-f84852 - - """ - result = False - polconn = __utils__["azurearm.get_client"]("policy", **kwargs) - try: - # pylint: disable=unused-variable - policy = polconn.policy_assignments.delete( - policy_assignment_name=name, scope=scope - ) - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - - return result - - -@_deprecation_message -def policy_assignment_create(name, scope, definition_name, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Create a policy assignment. - - :param name: The name of the policy assignment to create. - - :param scope: The scope of the policy assignment. - - :param definition_name: The name of the policy definition to assign. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.policy_assignment_create testassign \ - /subscriptions/bc75htn-a0fhsi-349b-56gh-4fghti-f84852 testpolicy - - """ - polconn = __utils__["azurearm.get_client"]("policy", **kwargs) - - # "get" doesn't work for built-in policies per https://github.com/Azure/azure-cli/issues/692 - # Uncomment this section when the ticket above is resolved. - # BEGIN - # definition = policy_definition_get( - # name=definition_name, - # **kwargs - # ) - # END - - # Delete this section when the ticket above is resolved. - # BEGIN - definition_list = policy_definitions_list(**kwargs) - if definition_name in definition_list: - definition = definition_list[definition_name] - else: - definition = { - "error": 'The policy definition named "{}" could not be found.'.format( - definition_name - ) - } - # END - - if "error" not in definition: - definition_id = str(definition["id"]) - - prop_kwargs = {"policy_definition_id": definition_id} - - policy_kwargs = kwargs.copy() - policy_kwargs.update(prop_kwargs) - - try: - policy_model = __utils__["azurearm.create_object_model"]( - "resource.policy", "PolicyAssignment", **policy_kwargs - ) - except TypeError as exc: - result = { - "error": "The object model could not be built. ({})".format(str(exc)) - } - return result - - try: - policy = polconn.policy_assignments.create( - scope=scope, policy_assignment_name=name, parameters=policy_model - ) - result = policy.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - else: - result = { - "error": 'The policy definition named "{}" could not be found.'.format( - definition_name - ) - } - - return result - - -@_deprecation_message -def policy_assignment_get(name, scope, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a specific policy assignment. - - :param name: The name of the policy assignment to query. - - :param scope: The scope of the policy assignment. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.policy_assignment_get testassign \ - /subscriptions/bc75htn-a0fhsi-349b-56gh-4fghti-f84852 - - """ - polconn = __utils__["azurearm.get_client"]("policy", **kwargs) - try: - policy = polconn.policy_assignments.get( - policy_assignment_name=name, scope=scope - ) - result = policy.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def policy_assignments_list_for_resource_group( - resource_group, **kwargs -): # pylint: disable=invalid-name - """ - .. versionadded:: 2019.2.0 - - List all policy assignments for a resource group. - - :param resource_group: The resource group name to list policy assignments within. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.policy_assignments_list_for_resource_group testgroup - - """ - result = {} - polconn = __utils__["azurearm.get_client"]("policy", **kwargs) - try: - policy_assign = __utils__["azurearm.paged_object_to_list"]( - polconn.policy_assignments.list_for_resource_group( - resource_group_name=resource_group, filter=kwargs.get("filter") - ) - ) - - for assign in policy_assign: - result[assign["name"]] = assign - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def policy_assignments_list(**kwargs): - """ - .. versionadded:: 2019.2.0 - - List all policy assignments for a subscription. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.policy_assignments_list - - """ - result = {} - polconn = __utils__["azurearm.get_client"]("policy", **kwargs) - try: - policy_assign = __utils__["azurearm.paged_object_to_list"]( - polconn.policy_assignments.list() - ) - - for assign in policy_assign: - result[assign["name"]] = assign - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def policy_definition_create_or_update( - name, policy_rule, **kwargs -): # pylint: disable=invalid-name - """ - .. versionadded:: 2019.2.0 - - Create or update a policy definition. - - :param name: The name of the policy definition to create or update. - - :param policy_rule: A dictionary defining the - `policy rule `_. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.policy_definition_create_or_update testpolicy '{...rule definition..}' - - """ - if not isinstance(policy_rule, dict): - result = {"error": "The policy rule must be a dictionary!"} - return result - - polconn = __utils__["azurearm.get_client"]("policy", **kwargs) - - # Convert OrderedDict to dict - prop_kwargs = { - "policy_rule": salt.utils.json.loads(salt.utils.json.dumps(policy_rule)) - } - - policy_kwargs = kwargs.copy() - policy_kwargs.update(prop_kwargs) - - try: - policy_model = __utils__["azurearm.create_object_model"]( - "resource.policy", "PolicyDefinition", **policy_kwargs - ) - except TypeError as exc: - result = {"error": "The object model could not be built. ({})".format(str(exc))} - return result - - try: - policy = polconn.policy_definitions.create_or_update( - policy_definition_name=name, parameters=policy_model - ) - result = policy.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - except SerializationError as exc: - result = { - "error": "The object model could not be parsed. ({})".format(str(exc)) - } - - return result - - -@_deprecation_message -def policy_definition_delete(name, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Delete a policy definition. - - :param name: The name of the policy definition to delete. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.policy_definition_delete testpolicy - - """ - result = False - polconn = __utils__["azurearm.get_client"]("policy", **kwargs) - try: - # pylint: disable=unused-variable - policy = polconn.policy_definitions.delete(policy_definition_name=name) - result = True - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - - return result - - -@_deprecation_message -def policy_definition_get(name, **kwargs): - """ - .. versionadded:: 2019.2.0 - - Get details about a specific policy definition. - - :param name: The name of the policy definition to query. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.policy_definition_get testpolicy - - """ - polconn = __utils__["azurearm.get_client"]("policy", **kwargs) - try: - policy_def = polconn.policy_definitions.get(policy_definition_name=name) - result = policy_def.as_dict() - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result - - -@_deprecation_message -def policy_definitions_list(hide_builtin=False, **kwargs): - """ - .. versionadded:: 2019.2.0 - - List all policy definitions for a subscription. - - :param hide_builtin: Boolean which will filter out BuiltIn policy definitions from the result. - - CLI Example: - - .. code-block:: bash - - salt-call azurearm_resource.policy_definitions_list - - """ - result = {} - polconn = __utils__["azurearm.get_client"]("policy", **kwargs) - try: - policy_defs = __utils__["azurearm.paged_object_to_list"]( - polconn.policy_definitions.list() - ) - - for policy in policy_defs: - if not (hide_builtin and policy["policy_type"] == "BuiltIn"): - result[policy["name"]] = policy - except CloudError as exc: - __utils__["azurearm.log_cloud_error"]("resource", str(exc), **kwargs) - result = {"error": str(exc)} - - return result diff --git a/salt/pillar/azureblob.py b/salt/pillar/azureblob.py deleted file mode 100644 index 4c26ba14b29..00000000000 --- a/salt/pillar/azureblob.py +++ /dev/null @@ -1,465 +0,0 @@ -""" -Use Azure Blob as a Pillar source. - -.. versionadded:: 3001 - -:maintainer: -:maturity: new -:depends: - * `azure-storage-blob `_ >= 12.0.0 - -The Azure Blob ext_pillar can be configured with the following parameters: - -.. code-block:: yaml - - ext_pillar: - - azureblob: - container: 'test_container' - connection_string: 'connection_string' - multiple_env: False - environment: 'base' - blob_cache_expire: 30 - blob_sync_on_update: True - -:param container: The name of the target Azure Blob Container. - -:param connection_string: The connection string to use to access the specified Azure Blob Container. - -:param multiple_env: Specifies whether the pillar should interpret top level folders as pillar environments. - Defaults to false. - -:param environment: Specifies which environment the container represents when in single environment mode. Defaults - to 'base' and is ignored if multiple_env is set as True. - -:param blob_cache_expire: Specifies expiration time of the Azure Blob metadata cache file. Defaults to 30s. - -:param blob_sync_on_update: Specifies if the cache is synced on update. Defaults to True. - -""" - -import logging -import os -import pickle -import time -from copy import deepcopy - -import salt.utils.files -import salt.utils.hashutils -from salt.pillar import Pillar - -HAS_LIBS = False -try: - # pylint: disable=no-name-in-module - from azure.storage.blob import BlobServiceClient - - # pylint: enable=no-name-in-module - HAS_LIBS = True -except ImportError: - pass - - -__virtualname__ = "azureblob" - -# Set up logging -log = logging.getLogger(__name__) - - -def __virtual__(): - if not HAS_LIBS: - return ( - False, - "The following dependency is required to use the Azure Blob ext_pillar: " - "Microsoft Azure Storage Blob >= 12.0.0 ", - ) - - return __virtualname__ - - -def ext_pillar( - minion_id, - pillar, # pylint: disable=W0613 - container, - connection_string, - multiple_env=False, - environment="base", - blob_cache_expire=30, - blob_sync_on_update=True, -): - """ - Execute a command and read the output as YAML. - - :param container: The name of the target Azure Blob Container. - - :param connection_string: The connection string to use to access the specified Azure Blob Container. - - :param multiple_env: Specifies whether the pillar should interpret top level folders as pillar environments. - Defaults to false. - - :param environment: Specifies which environment the container represents when in single environment mode. Defaults - to 'base' and is ignored if multiple_env is set as True. - - :param blob_cache_expire: Specifies expiration time of the Azure Blob metadata cache file. Defaults to 30s. - - :param blob_sync_on_update: Specifies if the cache is synced on update. Defaults to True. - - """ - # normpath is needed to remove appended '/' if root is empty string. - pillar_dir = os.path.normpath( - os.path.join(_get_cache_dir(), environment, container) - ) - - if __opts__["pillar_roots"].get(environment, []) == [pillar_dir]: - return {} - - metadata = _init( - connection_string, container, multiple_env, environment, blob_cache_expire - ) - - log.debug("Blob metadata: %s", metadata) - - if blob_sync_on_update: - # sync the containers to the local cache - log.info("Syncing local pillar cache from Azure Blob...") - for saltenv, env_meta in metadata.items(): - for container, files in _find_files(env_meta).items(): - for file_path in files: - cached_file_path = _get_cached_file_name( - container, saltenv, file_path - ) - log.info("%s - %s : %s", container, saltenv, file_path) - # load the file from Azure Blob if not in the cache or too old - _get_file_from_blob( - connection_string, - metadata, - saltenv, - container, - file_path, - cached_file_path, - ) - - log.info("Sync local pillar cache from Azure Blob completed.") - - opts = deepcopy(__opts__) - opts["pillar_roots"][environment] = ( - [os.path.join(pillar_dir, environment)] if multiple_env else [pillar_dir] - ) - - # Avoid recursively re-adding this same pillar - opts["ext_pillar"] = [x for x in opts["ext_pillar"] if "azureblob" not in x] - - pil = Pillar(opts, __grains__, minion_id, environment) - - compiled_pillar = pil.compile_pillar(ext=False) - - return compiled_pillar - - -def _init(connection_string, container, multiple_env, environment, blob_cache_expire): - """ - .. versionadded:: 3001 - - Connect to Blob Storage and download the metadata for each file in all containers specified and - cache the data to disk. - - :param connection_string: The connection string to use to access the specified Azure Blob Container. - - :param container: The name of the target Azure Blob Container. - - :param multiple_env: Specifies whether the pillar should interpret top level folders as pillar environments. - Defaults to false. - - :param environment: Specifies which environment the container represents when in single environment mode. Defaults - to 'base' and is ignored if multiple_env is set as True. - - :param blob_cache_expire: Specifies expiration time of the Azure Blob metadata cache file. Defaults to 30s. - - """ - cache_file = _get_containers_cache_filename(container) - exp = time.time() - blob_cache_expire - - # Check if cache_file exists and its mtime - if os.path.isfile(cache_file): - cache_file_mtime = os.path.getmtime(cache_file) - else: - # If the file does not exist then set mtime to 0 (aka epoch) - cache_file_mtime = 0 - - expired = cache_file_mtime <= exp - - log.debug( - "Blob storage container cache file %s is %sexpired, mtime_diff=%ss," - " expiration=%ss", - cache_file, - "" if expired else "not ", - cache_file_mtime - exp, - blob_cache_expire, - ) - - if expired: - pillars = _refresh_containers_cache_file( - connection_string, container, cache_file, multiple_env, environment - ) - else: - pillars = _read_containers_cache_file(cache_file) - - log.debug("Blob container retrieved pillars %s", pillars) - - return pillars - - -def _get_cache_dir(): - """ - .. versionadded:: 3001 - - Get pillar cache directory. Initialize it if it does not exist. - - """ - cache_dir = os.path.join(__opts__["cachedir"], "pillar_azureblob") - - if not os.path.isdir(cache_dir): - log.debug("Initializing Azure Blob Pillar Cache") - os.makedirs(cache_dir) - - return cache_dir - - -def _get_cached_file_name(container, saltenv, path): - """ - .. versionadded:: 3001 - - Return the cached file name for a container path file. - - :param container: The name of the target Azure Blob Container. - - :param saltenv: Specifies which environment the container represents. - - :param path: The path of the file in the container. - - """ - file_path = os.path.join(_get_cache_dir(), saltenv, container, path) - - # make sure container and saltenv directories exist - if not os.path.exists(os.path.dirname(file_path)): - os.makedirs(os.path.dirname(file_path)) - - return file_path - - -def _get_containers_cache_filename(container): - """ - .. versionadded:: 3001 - - Return the filename of the cache for container contents. Create the path if it does not exist. - - :param container: The name of the target Azure Blob Container. - - """ - cache_dir = _get_cache_dir() - if not os.path.exists(cache_dir): - os.makedirs(cache_dir) - - return os.path.join(cache_dir, "{}-files.cache".format(container)) - - -def _refresh_containers_cache_file( - connection_string, container, cache_file, multiple_env=False, environment="base" -): - """ - .. versionadded:: 3001 - - Downloads the entire contents of an Azure storage container to the local filesystem. - - :param connection_string: The connection string to use to access the specified Azure Blob Container. - - :param container: The name of the target Azure Blob Container. - - :param cache_file: The path of where the file will be cached. - - :param multiple_env: Specifies whether the pillar should interpret top level folders as pillar environments. - - :param environment: Specifies which environment the container represents when in single environment mode. This is - ignored if multiple_env is set as True. - - """ - try: - # Create the BlobServiceClient object which will be used to create a container client - blob_service_client = BlobServiceClient.from_connection_string( - connection_string - ) - - # Create the ContainerClient object - container_client = blob_service_client.get_container_client(container) - except Exception as exc: # pylint: disable=broad-except - log.error("Exception: %s", exc) - return False - - metadata = {} - - def _walk_blobs(saltenv="base", prefix=None): - # Walk the blobs in the container with a generator - blob_list = container_client.walk_blobs(name_starts_with=prefix) - - # Iterate over the generator - while True: - try: - blob = next(blob_list) - except StopIteration: - break - - log.debug("Raw blob attributes: %s", blob) - - # Directories end with "/". - if blob.name.endswith("/"): - # Recurse into the directory - _walk_blobs(prefix=blob.name) - continue - - if multiple_env: - saltenv = "base" if (not prefix or prefix == ".") else prefix[:-1] - - if saltenv not in metadata: - metadata[saltenv] = {} - - if container not in metadata[saltenv]: - metadata[saltenv][container] = [] - - metadata[saltenv][container].append(blob) - - _walk_blobs(saltenv=environment) - - # write the metadata to disk - if os.path.isfile(cache_file): - os.remove(cache_file) - - log.debug("Writing Azure blobs pillar cache file") - - with salt.utils.files.fopen(cache_file, "wb") as fp_: - pickle.dump(metadata, fp_) - - return metadata - - -def _read_containers_cache_file(cache_file): - """ - .. versionadded:: 3001 - - Return the contents of the containers cache file. - - :param cache_file: The path for where the file will be cached. - - """ - log.debug("Reading containers cache file") - - with salt.utils.files.fopen(cache_file, "rb") as fp_: - data = pickle.load(fp_) - - return data - - -def _find_files(metadata): - """ - .. versionadded:: 3001 - - Looks for all the files in the Azure Blob container cache metadata. - - :param metadata: The metadata for the container files. - - """ - ret = {} - - for container, data in metadata.items(): - if container not in ret: - ret[container] = [] - - # grab the paths from the metadata - file_paths = [k["name"] for k in data] - # filter out the dirs - ret[container] += [k for k in file_paths if not k.endswith("/")] - - return ret - - -def _find_file_meta(metadata, container, saltenv, path): - """ - .. versionadded:: 3001 - - Looks for a file's metadata in the Azure Blob Container cache file. - - :param metadata: The metadata for the container files. - - :param container: The name of the target Azure Blob Container. - - :param saltenv: Specifies which environment the container represents. - - :param path: The path of the file in the container. - - """ - env_meta = metadata[saltenv] if saltenv in metadata else {} - container_meta = env_meta[container] if container in env_meta else {} - - for item_meta in container_meta: - item_meta = dict(item_meta) - if "name" in item_meta and item_meta["name"] == path: - return item_meta - - -def _get_file_from_blob( - connection_string, metadata, saltenv, container, path, cached_file_path -): - """ - .. versionadded:: 3001 - - Downloads the entire contents of an Azure storage container to the local filesystem. - - :param connection_string: The connection string to use to access the specified Azure Blob Container. - - :param metadata: The metadata for the container files. - - :param saltenv: Specifies which environment the container represents when in single environment mode. This is - ignored if multiple_env is set as True. - - :param container: The name of the target Azure Blob Container. - - :param path: The path of the file in the container. - - :param cached_file_path: The path of where the file will be cached. - - """ - # check the local cache... - if os.path.isfile(cached_file_path): - file_meta = _find_file_meta(metadata, container, saltenv, path) - file_md5 = ( - "".join(list(filter(str.isalnum, file_meta["etag"]))) if file_meta else None - ) - - cached_md5 = salt.utils.hashutils.get_hash(cached_file_path, "md5") - - # hashes match we have a cache hit - log.debug( - "Cached file: path=%s, md5=%s, etag=%s", - cached_file_path, - cached_md5, - file_md5, - ) - if cached_md5 == file_md5: - return - - try: - # Create the BlobServiceClient object which will be used to create a container client - blob_service_client = BlobServiceClient.from_connection_string( - connection_string - ) - - # Create the ContainerClient object - container_client = blob_service_client.get_container_client(container) - - # Create the BlobClient object - blob_client = container_client.get_blob_client(path) - except Exception as exc: # pylint: disable=broad-except - log.error("Exception: %s", exc) - return False - - with salt.utils.files.fopen(cached_file_path, "wb") as outfile: - outfile.write(blob_client.download_blob().readall()) - - return diff --git a/salt/states/azurearm_compute.py b/salt/states/azurearm_compute.py deleted file mode 100644 index e23461afd27..00000000000 --- a/salt/states/azurearm_compute.py +++ /dev/null @@ -1,362 +0,0 @@ -""" -Azure (ARM) Compute State Module - -.. versionadded:: 2019.2.0 - -.. warning:: - - This cloud provider will be removed from Salt in version 3007 in favor of - the `saltext.azurerm Salt Extension - `_ - -:maintainer: -:maturity: new -:depends: - * `azure `_ >= 2.0.0 - * `azure-common `_ >= 1.1.8 - * `azure-mgmt `_ >= 1.0.0 - * `azure-mgmt-compute `_ >= 1.0.0 - * `azure-mgmt-network `_ >= 1.7.1 - * `azure-mgmt-resource `_ >= 1.1.0 - * `azure-mgmt-storage `_ >= 1.0.0 - * `azure-mgmt-web `_ >= 0.32.0 - * `azure-storage `_ >= 0.34.3 - * `msrestazure `_ >= 0.4.21 -:platform: linux - -:configuration: This module requires Azure Resource Manager credentials to be passed as a dictionary of - keyword arguments to the ``connection_auth`` parameter in order to work properly. Since the authentication - parameters are sensitive, it's recommended to pass them to the states via pillar. - - Required provider parameters: - - if using username and password: - * ``subscription_id`` - * ``username`` - * ``password`` - - if using a service principal: - * ``subscription_id`` - * ``tenant`` - * ``client_id`` - * ``secret`` - - Optional provider parameters: - - **cloud_environment**: Used to point the cloud driver to different API endpoints, such as Azure GovCloud. Possible values: - * ``AZURE_PUBLIC_CLOUD`` (default) - * ``AZURE_CHINA_CLOUD`` - * ``AZURE_US_GOV_CLOUD`` - * ``AZURE_GERMAN_CLOUD`` - - Example Pillar for Azure Resource Manager authentication: - - .. code-block:: yaml - - azurearm: - user_pass_auth: - subscription_id: 3287abc8-f98a-c678-3bde-326766fd3617 - username: fletch - password: 123pass - mysubscription: - subscription_id: 3287abc8-f98a-c678-3bde-326766fd3617 - tenant: ABCDEFAB-1234-ABCD-1234-ABCDEFABCDEF - client_id: ABCDEFAB-1234-ABCD-1234-ABCDEFABCDEF - secret: XXXXXXXXXXXXXXXXXXXXXXXX - cloud_environment: AZURE_PUBLIC_CLOUD - - Example states using Azure Resource Manager authentication: - - .. code-block:: jinja - - {% set profile = salt['pillar.get']('azurearm:mysubscription') %} - Ensure availability set exists: - azurearm_compute.availability_set_present: - - name: my_avail_set - - resource_group: my_rg - - virtual_machines: - - my_vm1 - - my_vm2 - - tags: - how_awesome: very - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - Ensure availability set is absent: - azurearm_compute.availability_set_absent: - - name: other_avail_set - - resource_group: my_rg - - connection_auth: {{ profile }} - -""" - -# Python libs - -import logging -from functools import wraps - -import salt.utils.azurearm - -__virtualname__ = "azurearm_compute" - -log = logging.getLogger(__name__) - - -def __virtual__(): - """ - Only make this state available if the azurearm_compute module is available. - """ - if "azurearm_compute.availability_set_create_or_update" in __salt__: - return __virtualname__ - return (False, "azurearm module could not be loaded") - - -def _deprecation_message(function): - """ - Decorator wrapper to warn about azurearm deprecation - """ - - @wraps(function) - def wrapped(*args, **kwargs): - salt.utils.versions.warn_until( - "Chlorine", - "The 'azurearm' functionality in Salt has been deprecated and its " - "functionality will be removed in version 3007 in favor of the " - "saltext.azurerm Salt Extension. " - "(https://github.com/salt-extensions/saltext-azurerm)", - category=FutureWarning, - ) - ret = function(*args, **salt.utils.args.clean_kwargs(**kwargs)) - return ret - - return wrapped - - -@_deprecation_message -def availability_set_present( - name, - resource_group, - tags=None, - platform_update_domain_count=None, - platform_fault_domain_count=None, - virtual_machines=None, - sku=None, - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure an availability set exists. - - :param name: - Name of the availability set. - - :param resource_group: - The resource group assigned to the availability set. - - :param tags: - A dictionary of strings can be passed as tag metadata to the availability set object. - - :param platform_update_domain_count: - An optional parameter which indicates groups of virtual machines and underlying physical hardware that can be - rebooted at the same time. - - :param platform_fault_domain_count: - An optional parameter which defines the group of virtual machines that share a common power source and network - switch. - - :param virtual_machines: - A list of names of existing virtual machines to be included in the availability set. - - :param sku: - The availability set SKU, which specifies whether the availability set is managed or not. Possible values are - 'Aligned' or 'Classic'. An 'Aligned' availability set is managed, 'Classic' is not. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure availability set exists: - azurearm_compute.availability_set_present: - - name: aset1 - - resource_group: group1 - - platform_update_domain_count: 5 - - platform_fault_domain_count: 3 - - sku: aligned - - tags: - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - require: - - azurearm_resource: Ensure resource group exists - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - if sku: - sku = {"name": sku.capitalize()} - - aset = __salt__["azurearm_compute.availability_set_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" not in aset: - tag_changes = __utils__["dictdiffer.deep_diff"]( - aset.get("tags", {}), tags or {} - ) - if tag_changes: - ret["changes"]["tags"] = tag_changes - - if platform_update_domain_count and ( - int(platform_update_domain_count) - != aset.get("platform_update_domain_count") - ): - ret["changes"]["platform_update_domain_count"] = { - "old": aset.get("platform_update_domain_count"), - "new": platform_update_domain_count, - } - - if platform_fault_domain_count and ( - int(platform_fault_domain_count) != aset.get("platform_fault_domain_count") - ): - ret["changes"]["platform_fault_domain_count"] = { - "old": aset.get("platform_fault_domain_count"), - "new": platform_fault_domain_count, - } - - if sku and (sku["name"] != aset.get("sku", {}).get("name")): - ret["changes"]["sku"] = {"old": aset.get("sku"), "new": sku} - - if virtual_machines: - if not isinstance(virtual_machines, list): - ret["comment"] = "Virtual machines must be supplied as a list!" - return ret - aset_vms = aset.get("virtual_machines", []) - remote_vms = sorted( - vm["id"].split("/")[-1].lower() for vm in aset_vms if "id" in aset_vms - ) - local_vms = sorted(vm.lower() for vm in virtual_machines or []) - if local_vms != remote_vms: - ret["changes"]["virtual_machines"] = { - "old": aset_vms, - "new": virtual_machines, - } - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Availability set {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["result"] = None - ret["comment"] = "Availability set {} would be updated.".format(name) - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "virtual_machines": virtual_machines, - "platform_update_domain_count": platform_update_domain_count, - "platform_fault_domain_count": platform_fault_domain_count, - "sku": sku, - "tags": tags, - }, - } - - if __opts__["test"]: - ret["comment"] = "Availability set {} would be created.".format(name) - ret["result"] = None - return ret - - aset_kwargs = kwargs.copy() - aset_kwargs.update(connection_auth) - - aset = __salt__["azurearm_compute.availability_set_create_or_update"]( - name=name, - resource_group=resource_group, - virtual_machines=virtual_machines, - platform_update_domain_count=platform_update_domain_count, - platform_fault_domain_count=platform_fault_domain_count, - sku=sku, - tags=tags, - **aset_kwargs - ) - - if "error" not in aset: - ret["result"] = True - ret["comment"] = "Availability set {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create availability set {}! ({})".format( - name, aset.get("error") - ) - return ret - - -@_deprecation_message -def availability_set_absent(name, resource_group, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure an availability set does not exist in a resource group. - - :param name: - Name of the availability set. - - :param resource_group: - Name of the resource group containing the availability set. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - aset = __salt__["azurearm_compute.availability_set_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" in aset: - ret["result"] = True - ret["comment"] = "Availability set {} was not found.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "Availability set {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": aset, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_compute.availability_set_delete"]( - name, resource_group, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Availability set {} has been deleted.".format(name) - ret["changes"] = {"old": aset, "new": {}} - return ret - - ret["comment"] = "Failed to delete availability set {}!".format(name) - return ret diff --git a/salt/states/azurearm_dns.py b/salt/states/azurearm_dns.py deleted file mode 100644 index 90f8c0b61c8..00000000000 --- a/salt/states/azurearm_dns.py +++ /dev/null @@ -1,762 +0,0 @@ -""" -Azure (ARM) DNS State Module - -.. versionadded:: 3000 - -.. warning:: - - This cloud provider will be removed from Salt in version 3007 in favor of - the `saltext.azurerm Salt Extension - `_ - -:maintainer: -:maturity: new -:depends: - * `azure `_ >= 2.0.0 - * `azure-common `_ >= 1.1.8 - * `azure-mgmt `_ >= 1.0.0 - * `azure-mgmt-compute `_ >= 1.0.0 - * `azure-mgmt-dns `_ >= 1.0.1 - * `azure-mgmt-network `_ >= 1.7.1 - * `azure-mgmt-resource `_ >= 1.1.0 - * `azure-mgmt-storage `_ >= 1.0.0 - * `azure-mgmt-web `_ >= 0.32.0 - * `azure-storage `_ >= 0.34.3 - * `msrestazure `_ >= 0.4.21 - -:platform: linux - -:configuration: - This module requires Azure Resource Manager credentials to be passed as a dictionary of - keyword arguments to the ``connection_auth`` parameter in order to work properly. Since the authentication - parameters are sensitive, it's recommended to pass them to the states via pillar. - -Required provider parameters: - - if using username and password: - - * ``subscription_id`` - * ``username`` - * ``password`` - - if using a service principal: - - * ``subscription_id`` - * ``tenant`` - * ``client_id`` - * ``secret`` - -Optional provider parameters: - - **cloud_environment**: Used to point the cloud driver to different API endpoints, such as Azure GovCloud. Possible values: - - Possible values: - - * ``AZURE_PUBLIC_CLOUD`` (default) - * ``AZURE_CHINA_CLOUD`` - * ``AZURE_US_GOV_CLOUD`` - * ``AZURE_GERMAN_CLOUD`` - - Example Pillar for Azure Resource Manager authentication: - - .. code-block:: yaml - - azurearm: - user_pass_auth: - subscription_id: 3287abc8-f98a-c678-3bde-326766fd3617 - username: fletch - password: 123pass - mysubscription: - subscription_id: 3287abc8-f98a-c678-3bde-326766fd3617 - tenant: ABCDEFAB-1234-ABCD-1234-ABCDEFABCDEF - client_id: ABCDEFAB-1234-ABCD-1234-ABCDEFABCDEF - secret: XXXXXXXXXXXXXXXXXXXXXXXX - cloud_environment: AZURE_PUBLIC_CLOUD - - Example states using Azure Resource Manager authentication: - - .. code-block:: none - - {% set profile = salt['pillar.get']('azurearm:mysubscription') %} - Ensure DNS zone exists: - azurearm_dns.zone_present: - - name: contoso.com - - resource_group: my_rg - - tags: - how_awesome: very - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - Ensure DNS record set exists: - azurearm_dns.record_set_present: - - name: web - - zone_name: contoso.com - - resource_group: my_rg - - record_type: A - - ttl: 300 - - arecords: - - ipv4_address: 10.0.0.1 - - tags: - how_awesome: very - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - Ensure DNS record set is absent: - azurearm_dns.record_set_absent: - - name: web - - zone_name: contoso.com - - resource_group: my_rg - - record_type: A - - connection_auth: {{ profile }} - - Ensure DNS zone is absent: - azurearm_dns.zone_absent: - - name: contoso.com - - resource_group: my_rg - - connection_auth: {{ profile }} - -""" -import logging -from functools import wraps - -import salt.utils.azurearm - -__virtualname__ = "azurearm_dns" - -log = logging.getLogger(__name__) - - -def __virtual__(): - """ - Only make this state available if the azurearm_dns module is available. - """ - if "azurearm_dns.zones_list_by_resource_group" in __salt__: - return __virtualname__ - return (False, "azurearm_dns module could not be loaded") - - -def _deprecation_message(function): - """ - Decorator wrapper to warn about azurearm deprecation - """ - - @wraps(function) - def wrapped(*args, **kwargs): - salt.utils.versions.warn_until( - "Chlorine", - "The 'azurearm' functionality in Salt has been deprecated and its " - "functionality will be removed in version 3007 in favor of the " - "saltext.azurerm Salt Extension. " - "(https://github.com/salt-extensions/saltext-azurerm)", - category=FutureWarning, - ) - ret = function(*args, **salt.utils.args.clean_kwargs(**kwargs)) - return ret - - return wrapped - - -@_deprecation_message -def zone_present( - name, - resource_group, - etag=None, - if_match=None, - if_none_match=None, - registration_virtual_networks=None, - resolution_virtual_networks=None, - tags=None, - zone_type="Public", - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 3000 - - Ensure a DNS zone exists. - - :param name: - Name of the DNS zone (without a terminating dot). - - :param resource_group: - The resource group assigned to the DNS zone. - - :param etag: - The etag of the zone. `Etags `_ are used - to handle concurrent changes to the same resource safely. - - :param if_match: - The etag of the DNS zone. Omit this value to always overwrite the current zone. Specify the last-seen etag - value to prevent accidentally overwritting any concurrent changes. - - :param if_none_match: - Set to '*' to allow a new DNS zone to be created, but to prevent updating an existing zone. Other values will - be ignored. - - :param registration_virtual_networks: - A list of references to virtual networks that register hostnames in this DNS zone. This is only when zone_type - is Private. (requires `azure-mgmt-dns `_ >= 2.0.0rc1) - - :param resolution_virtual_networks: - A list of references to virtual networks that resolve records in this DNS zone. This is only when zone_type is - Private. (requires `azure-mgmt-dns `_ >= 2.0.0rc1) - - :param tags: - A dictionary of strings can be passed as tag metadata to the DNS zone object. - - :param zone_type: - The type of this DNS zone (Public or Private). Possible values include: 'Public', 'Private'. Default value: 'Public' - (requires `azure-mgmt-dns `_ >= 2.0.0rc1) - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure DNS zone exists: - azurearm_dns.zone_present: - - name: contoso.com - - resource_group: my_rg - - zone_type: Private - - registration_virtual_networks: - - /subscriptions/{{ sub }}/resourceGroups/my_rg/providers/Microsoft.Network/virtualNetworks/test_vnet - - tags: - how_awesome: very - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - zone = __salt__["azurearm_dns.zone_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" not in zone: - tag_changes = __utils__["dictdiffer.deep_diff"]( - zone.get("tags", {}), tags or {} - ) - if tag_changes: - ret["changes"]["tags"] = tag_changes - - # The zone_type parameter is only accessible in azure-mgmt-dns >=2.0.0rc1 - if zone.get("zone_type"): - if zone.get("zone_type").lower() != zone_type.lower(): - ret["changes"]["zone_type"] = { - "old": zone["zone_type"], - "new": zone_type, - } - - if zone_type.lower() == "private": - # The registration_virtual_networks parameter is only accessible in azure-mgmt-dns >=2.0.0rc1 - if registration_virtual_networks and not isinstance( - registration_virtual_networks, list - ): - ret["comment"] = ( - "registration_virtual_networks must be supplied as a list of" - " VNET ID paths!" - ) - return ret - reg_vnets = zone.get("registration_virtual_networks", []) - remote_reg_vnets = sorted( - vnet["id"].lower() for vnet in reg_vnets if "id" in vnet - ) - local_reg_vnets = sorted( - vnet.lower() for vnet in registration_virtual_networks or [] - ) - if local_reg_vnets != remote_reg_vnets: - ret["changes"]["registration_virtual_networks"] = { - "old": remote_reg_vnets, - "new": local_reg_vnets, - } - - # The resolution_virtual_networks parameter is only accessible in azure-mgmt-dns >=2.0.0rc1 - if resolution_virtual_networks and not isinstance( - resolution_virtual_networks, list - ): - ret["comment"] = ( - "resolution_virtual_networks must be supplied as a list of VNET" - " ID paths!" - ) - return ret - res_vnets = zone.get("resolution_virtual_networks", []) - remote_res_vnets = sorted( - vnet["id"].lower() for vnet in res_vnets if "id" in vnet - ) - local_res_vnets = sorted( - vnet.lower() for vnet in resolution_virtual_networks or [] - ) - if local_res_vnets != remote_res_vnets: - ret["changes"]["resolution_virtual_networks"] = { - "old": remote_res_vnets, - "new": local_res_vnets, - } - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "DNS zone {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["result"] = None - ret["comment"] = "DNS zone {} would be updated.".format(name) - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "resource_group": resource_group, - "etag": etag, - "registration_virtual_networks": registration_virtual_networks, - "resolution_virtual_networks": resolution_virtual_networks, - "tags": tags, - "zone_type": zone_type, - }, - } - - if __opts__["test"]: - ret["comment"] = "DNS zone {} would be created.".format(name) - ret["result"] = None - return ret - - zone_kwargs = kwargs.copy() - zone_kwargs.update(connection_auth) - - zone = __salt__["azurearm_dns.zone_create_or_update"]( - name=name, - resource_group=resource_group, - etag=etag, - if_match=if_match, - if_none_match=if_none_match, - registration_virtual_networks=registration_virtual_networks, - resolution_virtual_networks=resolution_virtual_networks, - tags=tags, - zone_type=zone_type, - **zone_kwargs - ) - - if "error" not in zone: - ret["result"] = True - ret["comment"] = "DNS zone {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create DNS zone {}! ({})".format( - name, zone.get("error") - ) - return ret - - -@_deprecation_message -def zone_absent(name, resource_group, connection_auth=None): - """ - .. versionadded:: 3000 - - Ensure a DNS zone does not exist in the resource group. - - :param name: - Name of the DNS zone. - - :param resource_group: - The resource group assigned to the DNS zone. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - zone = __salt__["azurearm_dns.zone_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" in zone: - ret["result"] = True - ret["comment"] = "DNS zone {} was not found.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "DNS zone {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": zone, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_dns.zone_delete"]( - name, resource_group, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "DNS zone {} has been deleted.".format(name) - ret["changes"] = {"old": zone, "new": {}} - return ret - - ret["comment"] = "Failed to delete DNS zone {}!".format(name) - return ret - - -@_deprecation_message -def record_set_present( - name, - zone_name, - resource_group, - record_type, - if_match=None, - if_none_match=None, - etag=None, - metadata=None, - ttl=None, - arecords=None, - aaaa_records=None, - mx_records=None, - ns_records=None, - ptr_records=None, - srv_records=None, - txt_records=None, - cname_record=None, - soa_record=None, - caa_records=None, - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 3000 - - Ensure a record set exists in a DNS zone. - - :param name: - The name of the record set, relative to the name of the zone. - - :param zone_name: - Name of the DNS zone (without a terminating dot). - - :param resource_group: - The resource group assigned to the DNS zone. - - :param record_type: - The type of DNS record in this record set. Record sets of type SOA can be updated but not created - (they are created when the DNS zone is created). Possible values include: 'A', 'AAAA', 'CAA', 'CNAME', - 'MX', 'NS', 'PTR', 'SOA', 'SRV', 'TXT' - - :param if_match: - The etag of the record set. Omit this value to always overwrite the current record set. Specify the last-seen - etag value to prevent accidentally overwritting any concurrent changes. - - :param if_none_match: - Set to '*' to allow a new record set to be created, but to prevent updating an existing record set. Other values - will be ignored. - - :param etag: - The etag of the record set. `Etags `__ are - used to handle concurrent changes to the same resource safely. - - :param metadata: - A dictionary of strings can be passed as tag metadata to the record set object. - - :param ttl: - The TTL (time-to-live) of the records in the record set. Required when specifying record information. - - :param arecords: - The list of A records in the record set. View the - `Azure SDK documentation `__ - to create a list of dictionaries representing the record objects. - - :param aaaa_records: - The list of AAAA records in the record set. View the - `Azure SDK documentation `__ - to create a list of dictionaries representing the record objects. - - :param mx_records: - The list of MX records in the record set. View the - `Azure SDK documentation `__ - to create a list of dictionaries representing the record objects. - - :param ns_records: - The list of NS records in the record set. View the - `Azure SDK documentation `__ - to create a list of dictionaries representing the record objects. - - :param ptr_records: - The list of PTR records in the record set. View the - `Azure SDK documentation `__ - to create a list of dictionaries representing the record objects. - - :param srv_records: - The list of SRV records in the record set. View the - `Azure SDK documentation `__ - to create a list of dictionaries representing the record objects. - - :param txt_records: - The list of TXT records in the record set. View the - `Azure SDK documentation `__ - to create a list of dictionaries representing the record objects. - - :param cname_record: - The CNAME record in the record set. View the - `Azure SDK documentation `__ - to create a dictionary representing the record object. - - :param soa_record: - The SOA record in the record set. View the - `Azure SDK documentation `__ - to create a dictionary representing the record object. - - :param caa_records: - The list of CAA records in the record set. View the - `Azure SDK documentation `__ - to create a list of dictionaries representing the record objects. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure record set exists: - azurearm_dns.record_set_present: - - name: web - - zone_name: contoso.com - - resource_group: my_rg - - record_type: A - - ttl: 300 - - arecords: - - ipv4_address: 10.0.0.1 - - metadata: - how_awesome: very - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - record_vars = [ - "arecords", - "aaaa_records", - "mx_records", - "ns_records", - "ptr_records", - "srv_records", - "txt_records", - "cname_record", - "soa_record", - "caa_records", - ] - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - rec_set = __salt__["azurearm_dns.record_set_get"]( - name, - zone_name, - resource_group, - record_type, - azurearm_log_level="info", - **connection_auth - ) - - if "error" not in rec_set: - metadata_changes = __utils__["dictdiffer.deep_diff"]( - rec_set.get("metadata", {}), metadata or {} - ) - if metadata_changes: - ret["changes"]["metadata"] = metadata_changes - - for record_str in record_vars: - # pylint: disable=eval-used - record = eval(record_str) - if record: - if not ttl: - ret[ - "comment" - ] = "TTL is required when specifying record information!" - return ret - if not rec_set.get(record_str): - ret["changes"] = {"new": {record_str: record}} - continue - if record_str[-1] != "s": - if not isinstance(record, dict): - ret[ - "comment" - ] = "{} record information must be specified as a dictionary!".format( - record_str - ) - return ret - for k, v in record.items(): - if v != rec_set[record_str].get(k): - ret["changes"] = {"new": {record_str: record}} - elif record_str[-1] == "s": - if not isinstance(record, list): - ret["comment"] = ( - "{} record information must be specified as a list of" - " dictionaries!".format(record_str) - ) - return ret - local, remote = ( - sorted(config) for config in (record, rec_set[record_str]) - ) - for val in local: - for key in val: - local_val = val[key] - remote_val = remote.get(key) - if isinstance(local_val, str): - local_val = local_val.lower() - if isinstance(remote_val, str): - remote_val = remote_val.lower() - if local_val != remote_val: - ret["changes"] = {"new": {record_str: record}} - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Record set {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["result"] = None - ret["comment"] = "Record set {} would be updated.".format(name) - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "zone_name": zone_name, - "resource_group": resource_group, - "record_type": record_type, - "etag": etag, - "metadata": metadata, - "ttl": ttl, - }, - } - for record in record_vars: - # pylint: disable=eval-used - if eval(record): - # pylint: disable=eval-used - ret["changes"]["new"][record] = eval(record) - - if __opts__["test"]: - ret["comment"] = "Record set {} would be created.".format(name) - ret["result"] = None - return ret - - rec_set_kwargs = kwargs.copy() - rec_set_kwargs.update(connection_auth) - - rec_set = __salt__["azurearm_dns.record_set_create_or_update"]( - name=name, - zone_name=zone_name, - resource_group=resource_group, - record_type=record_type, - if_match=if_match, - if_none_match=if_none_match, - etag=etag, - ttl=ttl, - metadata=metadata, - arecords=arecords, - aaaa_records=aaaa_records, - mx_records=mx_records, - ns_records=ns_records, - ptr_records=ptr_records, - srv_records=srv_records, - txt_records=txt_records, - cname_record=cname_record, - soa_record=soa_record, - caa_records=caa_records, - **rec_set_kwargs - ) - - if "error" not in rec_set: - ret["result"] = True - ret["comment"] = "Record set {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create record set {}! ({})".format( - name, rec_set.get("error") - ) - return ret - - -@_deprecation_message -def record_set_absent(name, zone_name, resource_group, connection_auth=None): - """ - .. versionadded:: 3000 - - Ensure a record set does not exist in the DNS zone. - - :param name: - Name of the record set. - - :param zone_name: - Name of the DNS zone. - - :param resource_group: - The resource group assigned to the DNS zone. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - rec_set = __salt__["azurearm_dns.record_set_get"]( - name, zone_name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" in rec_set: - ret["result"] = True - ret["comment"] = "Record set {} was not found in zone {}.".format( - name, zone_name - ) - return ret - - elif __opts__["test"]: - ret["comment"] = "Record set {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": rec_set, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_dns.record_set_delete"]( - name, zone_name, resource_group, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Record set {} has been deleted.".format(name) - ret["changes"] = {"old": rec_set, "new": {}} - return ret - - ret["comment"] = "Failed to delete record set {}!".format(name) - return ret diff --git a/salt/states/azurearm_network.py b/salt/states/azurearm_network.py deleted file mode 100644 index 2555a2d06ad..00000000000 --- a/salt/states/azurearm_network.py +++ /dev/null @@ -1,2594 +0,0 @@ -""" -Azure (ARM) Network State Module - -.. versionadded:: 2019.2.0 - -.. warning:: - - This cloud provider will be removed from Salt in version 3007 in favor of - the `saltext.azurerm Salt Extension - `_ - -:maintainer: -:maturity: new -:depends: - * `azure `_ >= 2.0.0 - * `azure-common `_ >= 1.1.8 - * `azure-mgmt `_ >= 1.0.0 - * `azure-mgmt-compute `_ >= 1.0.0 - * `azure-mgmt-network `_ >= 1.7.1 - * `azure-mgmt-resource `_ >= 1.1.0 - * `azure-mgmt-storage `_ >= 1.0.0 - * `azure-mgmt-web `_ >= 0.32.0 - * `azure-storage `_ >= 0.34.3 - * `msrestazure `_ >= 0.4.21 -:platform: linux - -:configuration: This module requires Azure Resource Manager credentials to be passed as a dictionary of - keyword arguments to the ``connection_auth`` parameter in order to work properly. Since the authentication - parameters are sensitive, it's recommended to pass them to the states via pillar. - - Required provider parameters: - - if using username and password: - * ``subscription_id`` - * ``username`` - * ``password`` - - if using a service principal: - * ``subscription_id`` - * ``tenant`` - * ``client_id`` - * ``secret`` - - Optional provider parameters: - - **cloud_environment**: Used to point the cloud driver to different API endpoints, such as Azure GovCloud. Possible values: - * ``AZURE_PUBLIC_CLOUD`` (default) - * ``AZURE_CHINA_CLOUD`` - * ``AZURE_US_GOV_CLOUD`` - * ``AZURE_GERMAN_CLOUD`` - - Example Pillar for Azure Resource Manager authentication: - - .. code-block:: yaml - - azurearm: - user_pass_auth: - subscription_id: 3287abc8-f98a-c678-3bde-326766fd3617 - username: fletch - password: 123pass - mysubscription: - subscription_id: 3287abc8-f98a-c678-3bde-326766fd3617 - tenant: ABCDEFAB-1234-ABCD-1234-ABCDEFABCDEF - client_id: ABCDEFAB-1234-ABCD-1234-ABCDEFABCDEF - secret: XXXXXXXXXXXXXXXXXXXXXXXX - cloud_environment: AZURE_PUBLIC_CLOUD - - Example states using Azure Resource Manager authentication: - - .. code-block:: jinja - - {% set profile = salt['pillar.get']('azurearm:mysubscription') %} - Ensure virtual network exists: - azurearm_network.virtual_network_present: - - name: my_vnet - - resource_group: my_rg - - address_prefixes: - - '10.0.0.0/8' - - '192.168.0.0/16' - - dns_servers: - - '8.8.8.8' - - tags: - how_awesome: very - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - Ensure virtual network is absent: - azurearm_network.virtual_network_absent: - - name: other_vnet - - resource_group: my_rg - - connection_auth: {{ profile }} - -""" -import logging -from functools import wraps - -import salt.utils.azurearm - -__virtualname__ = "azurearm_network" - -log = logging.getLogger(__name__) - - -def __virtual__(): - """ - Only make this state available if the azurearm_network module is available. - """ - if "azurearm_network.check_ip_address_availability" in __salt__: - return __virtualname__ - return (False, "azurearm_network module could not be loaded") - - -def _deprecation_message(function): - """ - Decorator wrapper to warn about azurearm deprecation - """ - - @wraps(function) - def wrapped(*args, **kwargs): - salt.utils.versions.warn_until( - "Chlorine", - "The 'azurearm' functionality in Salt has been deprecated and its " - "functionality will be removed in version 3007 in favor of the " - "saltext.azurerm Salt Extension. " - "(https://github.com/salt-extensions/saltext-azurerm)", - category=FutureWarning, - ) - ret = function(*args, **salt.utils.args.clean_kwargs(**kwargs)) - return ret - - return wrapped - - -@_deprecation_message -def virtual_network_present( - name, - address_prefixes, - resource_group, - dns_servers=None, - tags=None, - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure a virtual network exists. - - :param name: - Name of the virtual network. - - :param resource_group: - The resource group assigned to the virtual network. - - :param address_prefixes: - A list of CIDR blocks which can be used by subnets within the virtual network. - - :param dns_servers: - A list of DNS server addresses. - - :param tags: - A dictionary of strings can be passed as tag metadata to the virtual network object. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure virtual network exists: - azurearm_network.virtual_network_present: - - name: vnet1 - - resource_group: group1 - - address_prefixes: - - '10.0.0.0/8' - - '192.168.0.0/16' - - dns_servers: - - '8.8.8.8' - - tags: - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - require: - - azurearm_resource: Ensure resource group exists - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - vnet = __salt__["azurearm_network.virtual_network_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" not in vnet: - tag_changes = __utils__["dictdiffer.deep_diff"]( - vnet.get("tags", {}), tags or {} - ) - if tag_changes: - ret["changes"]["tags"] = tag_changes - - dns_changes = set(dns_servers or []).symmetric_difference( - set(vnet.get("dhcp_options", {}).get("dns_servers", [])) - ) - if dns_changes: - ret["changes"]["dns_servers"] = { - "old": vnet.get("dhcp_options", {}).get("dns_servers", []), - "new": dns_servers, - } - - addr_changes = set(address_prefixes or []).symmetric_difference( - set(vnet.get("address_space", {}).get("address_prefixes", [])) - ) - if addr_changes: - ret["changes"]["address_space"] = { - "address_prefixes": { - "old": vnet.get("address_space", {}).get("address_prefixes", []), - "new": address_prefixes, - } - } - - if kwargs.get("enable_ddos_protection", False) != vnet.get( - "enable_ddos_protection" - ): - ret["changes"]["enable_ddos_protection"] = { - "old": vnet.get("enable_ddos_protection"), - "new": kwargs.get("enable_ddos_protection"), - } - - if kwargs.get("enable_vm_protection", False) != vnet.get( - "enable_vm_protection" - ): - ret["changes"]["enable_vm_protection"] = { - "old": vnet.get("enable_vm_protection"), - "new": kwargs.get("enable_vm_protection"), - } - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Virtual network {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["result"] = None - ret["comment"] = "Virtual network {} would be updated.".format(name) - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "resource_group": resource_group, - "address_space": {"address_prefixes": address_prefixes}, - "dhcp_options": {"dns_servers": dns_servers}, - "enable_ddos_protection": kwargs.get("enable_ddos_protection", False), - "enable_vm_protection": kwargs.get("enable_vm_protection", False), - "tags": tags, - }, - } - - if __opts__["test"]: - ret["comment"] = "Virtual network {} would be created.".format(name) - ret["result"] = None - return ret - - vnet_kwargs = kwargs.copy() - vnet_kwargs.update(connection_auth) - - vnet = __salt__["azurearm_network.virtual_network_create_or_update"]( - name=name, - resource_group=resource_group, - address_prefixes=address_prefixes, - dns_servers=dns_servers, - tags=tags, - **vnet_kwargs - ) - - if "error" not in vnet: - ret["result"] = True - ret["comment"] = "Virtual network {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create virtual network {}! ({})".format( - name, vnet.get("error") - ) - return ret - - -@_deprecation_message -def virtual_network_absent(name, resource_group, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure a virtual network does not exist in the resource group. - - :param name: - Name of the virtual network. - - :param resource_group: - The resource group assigned to the virtual network. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - vnet = __salt__["azurearm_network.virtual_network_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" in vnet: - ret["result"] = True - ret["comment"] = "Virtual network {} was not found.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "Virtual network {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": vnet, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_network.virtual_network_delete"]( - name, resource_group, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Virtual network {} has been deleted.".format(name) - ret["changes"] = {"old": vnet, "new": {}} - return ret - - ret["comment"] = "Failed to delete virtual network {}!".format(name) - return ret - - -@_deprecation_message -def subnet_present( - name, - address_prefix, - virtual_network, - resource_group, - security_group=None, - route_table=None, - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure a subnet exists. - - :param name: - Name of the subnet. - - :param address_prefix: - A CIDR block used by the subnet within the virtual network. - - :param virtual_network: - Name of the existing virtual network to contain the subnet. - - :param resource_group: - The resource group assigned to the virtual network. - - :param security_group: - The name of the existing network security group to assign to the subnet. - - :param route_table: - The name of the existing route table to assign to the subnet. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure subnet exists: - azurearm_network.subnet_present: - - name: vnet1_sn1 - - virtual_network: vnet1 - - resource_group: group1 - - address_prefix: '192.168.1.0/24' - - security_group: nsg1 - - route_table: rt1 - - connection_auth: {{ profile }} - - require: - - azurearm_network: Ensure virtual network exists - - azurearm_network: Ensure network security group exists - - azurearm_network: Ensure route table exists - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - snet = __salt__["azurearm_network.subnet_get"]( - name, - virtual_network, - resource_group, - azurearm_log_level="info", - **connection_auth - ) - - if "error" not in snet: - if address_prefix != snet.get("address_prefix"): - ret["changes"]["address_prefix"] = { - "old": snet.get("address_prefix"), - "new": address_prefix, - } - - nsg_name = None - if snet.get("network_security_group"): - nsg_name = snet["network_security_group"]["id"].split("/")[-1] - - if security_group and (security_group != nsg_name): - ret["changes"]["network_security_group"] = { - "old": nsg_name, - "new": security_group, - } - - rttbl_name = None - if snet.get("route_table"): - rttbl_name = snet["route_table"]["id"].split("/")[-1] - - if route_table and (route_table != rttbl_name): - ret["changes"]["route_table"] = {"old": rttbl_name, "new": route_table} - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Subnet {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["result"] = None - ret["comment"] = "Subnet {} would be updated.".format(name) - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "address_prefix": address_prefix, - "network_security_group": security_group, - "route_table": route_table, - }, - } - - if __opts__["test"]: - ret["comment"] = "Subnet {} would be created.".format(name) - ret["result"] = None - return ret - - snet_kwargs = kwargs.copy() - snet_kwargs.update(connection_auth) - - snet = __salt__["azurearm_network.subnet_create_or_update"]( - name=name, - virtual_network=virtual_network, - resource_group=resource_group, - address_prefix=address_prefix, - network_security_group=security_group, - route_table=route_table, - **snet_kwargs - ) - - if "error" not in snet: - ret["result"] = True - ret["comment"] = "Subnet {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create subnet {}! ({})".format(name, snet.get("error")) - return ret - - -@_deprecation_message -def subnet_absent(name, virtual_network, resource_group, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure a virtual network does not exist in the virtual network. - - :param name: - Name of the subnet. - - :param virtual_network: - Name of the existing virtual network containing the subnet. - - :param resource_group: - The resource group assigned to the virtual network. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - snet = __salt__["azurearm_network.subnet_get"]( - name, - virtual_network, - resource_group, - azurearm_log_level="info", - **connection_auth - ) - - if "error" in snet: - ret["result"] = True - ret["comment"] = "Subnet {} was not found.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "Subnet {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": snet, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_network.subnet_delete"]( - name, virtual_network, resource_group, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Subnet {} has been deleted.".format(name) - ret["changes"] = {"old": snet, "new": {}} - return ret - - ret["comment"] = "Failed to delete subnet {}!".format(name) - return ret - - -@_deprecation_message -def network_security_group_present( - name, resource_group, tags=None, security_rules=None, connection_auth=None, **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure a network security group exists. - - :param name: - Name of the network security group. - - :param resource_group: - The resource group assigned to the network security group. - - :param tags: - A dictionary of strings can be passed as tag metadata to the network security group object. - - :param security_rules: An optional list of dictionaries representing valid SecurityRule objects. See the - documentation for the security_rule_present state or security_rule_create_or_update execution module - for more information on required and optional parameters for security rules. The rules are only - managed if this parameter is present. When this parameter is absent, implemented rules will not be removed, - and will merely become unmanaged. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure network security group exists: - azurearm_network.network_security_group_present: - - name: nsg1 - - resource_group: group1 - - security_rules: - - name: nsg1_rule1 - priority: 100 - protocol: tcp - access: allow - direction: outbound - source_address_prefix: virtualnetwork - destination_address_prefix: internet - source_port_range: '*' - destination_port_range: '*' - - name: nsg1_rule2 - priority: 101 - protocol: tcp - access: allow - direction: inbound - source_address_prefix: internet - destination_address_prefix: virtualnetwork - source_port_range: '*' - destination_port_ranges: - - '80' - - '443' - - tags: - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - require: - - azurearm_resource: Ensure resource group exists - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - nsg = __salt__["azurearm_network.network_security_group_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" not in nsg: - tag_changes = __utils__["dictdiffer.deep_diff"](nsg.get("tags", {}), tags or {}) - if tag_changes: - ret["changes"]["tags"] = tag_changes - - if security_rules: - comp_ret = __utils__["azurearm.compare_list_of_dicts"]( - nsg.get("security_rules", []), security_rules - ) - - if comp_ret.get("comment"): - ret["comment"] = '"security_rules" {}'.format(comp_ret["comment"]) - return ret - - if comp_ret.get("changes"): - ret["changes"]["security_rules"] = comp_ret["changes"] - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Network security group {} is already present.".format( - name - ) - return ret - - if __opts__["test"]: - ret["result"] = None - ret["comment"] = "Network security group {} would be updated.".format(name) - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "resource_group": resource_group, - "tags": tags, - "security_rules": security_rules, - }, - } - - if __opts__["test"]: - ret["comment"] = "Network security group {} would be created.".format(name) - ret["result"] = None - return ret - - nsg_kwargs = kwargs.copy() - nsg_kwargs.update(connection_auth) - - nsg = __salt__["azurearm_network.network_security_group_create_or_update"]( - name=name, - resource_group=resource_group, - tags=tags, - security_rules=security_rules, - **nsg_kwargs - ) - - if "error" not in nsg: - ret["result"] = True - ret["comment"] = "Network security group {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create network security group {}! ({})".format( - name, nsg.get("error") - ) - return ret - - -@_deprecation_message -def network_security_group_absent(name, resource_group, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure a network security group does not exist in the resource group. - - :param name: - Name of the network security group. - - :param resource_group: - The resource group assigned to the network security group. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - nsg = __salt__["azurearm_network.network_security_group_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" in nsg: - ret["result"] = True - ret["comment"] = "Network security group {} was not found.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "Network security group {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": nsg, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_network.network_security_group_delete"]( - name, resource_group, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Network security group {} has been deleted.".format(name) - ret["changes"] = {"old": nsg, "new": {}} - return ret - - ret["comment"] = "Failed to delete network security group {}!".format(name) - return ret - - -@_deprecation_message -def security_rule_present( - name, - access, - direction, - priority, - protocol, - security_group, - resource_group, - destination_address_prefix=None, - destination_port_range=None, - source_address_prefix=None, - source_port_range=None, - description=None, - destination_address_prefixes=None, - destination_port_ranges=None, - source_address_prefixes=None, - source_port_ranges=None, - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure a security rule exists. - - :param name: - Name of the security rule. - - :param access: - 'allow' or 'deny' - - :param direction: - 'inbound' or 'outbound' - - :param priority: - Integer between 100 and 4096 used for ordering rule application. - - :param protocol: - 'tcp', 'udp', or '*' - - :param security_group: - The name of the existing network security group to contain the security rule. - - :param resource_group: - The resource group assigned to the network security group. - - :param description: - Optional description of the security rule. - - :param destination_address_prefix: - The CIDR or destination IP range. Asterix '*' can also be used to match all destination IPs. - Default tags such as 'VirtualNetwork', 'AzureLoadBalancer' and 'Internet' can also be used. - If this is an ingress rule, specifies where network traffic originates from. - - :param destination_port_range: - The destination port or range. Integer or range between 0 and 65535. Asterix '*' - can also be used to match all ports. - - :param source_address_prefix: - The CIDR or source IP range. Asterix '*' can also be used to match all source IPs. - Default tags such as 'VirtualNetwork', 'AzureLoadBalancer' and 'Internet' can also be used. - If this is an ingress rule, specifies where network traffic originates from. - - :param source_port_range: - The source port or range. Integer or range between 0 and 65535. Asterix '*' - can also be used to match all ports. - - :param destination_address_prefixes: - A list of destination_address_prefix values. This parameter overrides destination_address_prefix - and will cause any value entered there to be ignored. - - :param destination_port_ranges: - A list of destination_port_range values. This parameter overrides destination_port_range - and will cause any value entered there to be ignored. - - :param source_address_prefixes: - A list of source_address_prefix values. This parameter overrides source_address_prefix - and will cause any value entered there to be ignored. - - :param source_port_ranges: - A list of source_port_range values. This parameter overrides source_port_range - and will cause any value entered there to be ignored. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure security rule exists: - azurearm_network.security_rule_present: - - name: nsg1_rule2 - - security_group: nsg1 - - resource_group: group1 - - priority: 101 - - protocol: tcp - - access: allow - - direction: inbound - - source_address_prefix: internet - - destination_address_prefix: virtualnetwork - - source_port_range: '*' - - destination_port_ranges: - - '80' - - '443' - - connection_auth: {{ profile }} - - require: - - azurearm_network: Ensure network security group exists - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - exclusive_params = [ - ("source_port_ranges", "source_port_range"), - ("source_address_prefixes", "source_address_prefix"), - ("destination_port_ranges", "destination_port_range"), - ("destination_address_prefixes", "destination_address_prefix"), - ] - - for params in exclusive_params: - # pylint: disable=eval-used - if not eval(params[0]) and not eval(params[1]): - ret["comment"] = "Either the {} or {} parameter must be provided!".format( - params[0], params[1] - ) - return ret - # pylint: disable=eval-used - if eval(params[0]): - # pylint: disable=eval-used - if not isinstance(eval(params[0]), list): - ret["comment"] = "The {} parameter must be a list!".format(params[0]) - return ret - # pylint: disable=exec-used - exec("{} = None".format(params[1])) - - rule = __salt__["azurearm_network.security_rule_get"]( - name, - security_group, - resource_group, - azurearm_log_level="info", - **connection_auth - ) - - if "error" not in rule: - # access changes - if access.capitalize() != rule.get("access"): - ret["changes"]["access"] = {"old": rule.get("access"), "new": access} - - # description changes - if description != rule.get("description"): - ret["changes"]["description"] = { - "old": rule.get("description"), - "new": description, - } - - # direction changes - if direction.capitalize() != rule.get("direction"): - ret["changes"]["direction"] = { - "old": rule.get("direction"), - "new": direction, - } - - # priority changes - if int(priority) != rule.get("priority"): - ret["changes"]["priority"] = {"old": rule.get("priority"), "new": priority} - - # protocol changes - if protocol.lower() != rule.get("protocol", "").lower(): - ret["changes"]["protocol"] = {"old": rule.get("protocol"), "new": protocol} - - # destination_port_range changes - if destination_port_range != rule.get("destination_port_range"): - ret["changes"]["destination_port_range"] = { - "old": rule.get("destination_port_range"), - "new": destination_port_range, - } - - # source_port_range changes - if source_port_range != rule.get("source_port_range"): - ret["changes"]["source_port_range"] = { - "old": rule.get("source_port_range"), - "new": source_port_range, - } - - # destination_port_ranges changes - if sorted(destination_port_ranges or []) != sorted( - rule.get("destination_port_ranges", []) - ): - ret["changes"]["destination_port_ranges"] = { - "old": rule.get("destination_port_ranges"), - "new": destination_port_ranges, - } - - # source_port_ranges changes - if sorted(source_port_ranges or []) != sorted( - rule.get("source_port_ranges", []) - ): - ret["changes"]["source_port_ranges"] = { - "old": rule.get("source_port_ranges"), - "new": source_port_ranges, - } - - # destination_address_prefix changes - if (destination_address_prefix or "").lower() != rule.get( - "destination_address_prefix", "" - ).lower(): - ret["changes"]["destination_address_prefix"] = { - "old": rule.get("destination_address_prefix"), - "new": destination_address_prefix, - } - - # source_address_prefix changes - if (source_address_prefix or "").lower() != rule.get( - "source_address_prefix", "" - ).lower(): - ret["changes"]["source_address_prefix"] = { - "old": rule.get("source_address_prefix"), - "new": source_address_prefix, - } - - # destination_address_prefixes changes - if sorted(destination_address_prefixes or []) != sorted( - rule.get("destination_address_prefixes", []) - ): - if len(destination_address_prefixes or []) != len( - rule.get("destination_address_prefixes", []) - ): - ret["changes"]["destination_address_prefixes"] = { - "old": rule.get("destination_address_prefixes"), - "new": destination_address_prefixes, - } - else: - local_dst_addrs, remote_dst_addrs = ( - sorted(destination_address_prefixes), - sorted(rule.get("destination_address_prefixes")), - ) - for idx, val in enumerate(local_dst_addrs): - if val.lower() != remote_dst_addrs[idx].lower(): - ret["changes"]["destination_address_prefixes"] = { - "old": rule.get("destination_address_prefixes"), - "new": destination_address_prefixes, - } - break - - # source_address_prefixes changes - if sorted(source_address_prefixes or []) != sorted( - rule.get("source_address_prefixes", []) - ): - if len(source_address_prefixes or []) != len( - rule.get("source_address_prefixes", []) - ): - ret["changes"]["source_address_prefixes"] = { - "old": rule.get("source_address_prefixes"), - "new": source_address_prefixes, - } - else: - local_src_addrs, remote_src_addrs = ( - sorted(source_address_prefixes), - sorted(rule.get("source_address_prefixes")), - ) - for idx, val in enumerate(local_src_addrs): - if val.lower() != remote_src_addrs[idx].lower(): - ret["changes"]["source_address_prefixes"] = { - "old": rule.get("source_address_prefixes"), - "new": source_address_prefixes, - } - break - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Security rule {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["result"] = None - ret["comment"] = "Security rule {} would be updated.".format(name) - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "access": access, - "description": description, - "direction": direction, - "priority": priority, - "protocol": protocol, - "destination_address_prefix": destination_address_prefix, - "destination_address_prefixes": destination_address_prefixes, - "destination_port_range": destination_port_range, - "destination_port_ranges": destination_port_ranges, - "source_address_prefix": source_address_prefix, - "source_address_prefixes": source_address_prefixes, - "source_port_range": source_port_range, - "source_port_ranges": source_port_ranges, - }, - } - - if __opts__["test"]: - ret["comment"] = "Security rule {} would be created.".format(name) - ret["result"] = None - return ret - - rule_kwargs = kwargs.copy() - rule_kwargs.update(connection_auth) - - rule = __salt__["azurearm_network.security_rule_create_or_update"]( - name=name, - access=access, - description=description, - direction=direction, - priority=priority, - protocol=protocol, - security_group=security_group, - resource_group=resource_group, - destination_address_prefix=destination_address_prefix, - destination_address_prefixes=destination_address_prefixes, - destination_port_range=destination_port_range, - destination_port_ranges=destination_port_ranges, - source_address_prefix=source_address_prefix, - source_address_prefixes=source_address_prefixes, - source_port_range=source_port_range, - source_port_ranges=source_port_ranges, - **rule_kwargs - ) - - if "error" not in rule: - ret["result"] = True - ret["comment"] = "Security rule {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create security rule {}! ({})".format( - name, rule.get("error") - ) - return ret - - -@_deprecation_message -def security_rule_absent(name, security_group, resource_group, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure a security rule does not exist in the network security group. - - :param name: - Name of the security rule. - - :param security_group: - The network security group containing the security rule. - - :param resource_group: - The resource group assigned to the network security group. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - rule = __salt__["azurearm_network.security_rule_get"]( - name, - security_group, - resource_group, - azurearm_log_level="info", - **connection_auth - ) - - if "error" in rule: - ret["result"] = True - ret["comment"] = "Security rule {} was not found.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "Security rule {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": rule, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_network.security_rule_delete"]( - name, security_group, resource_group, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Security rule {} has been deleted.".format(name) - ret["changes"] = {"old": rule, "new": {}} - return ret - - ret["comment"] = "Failed to delete security rule {}!".format(name) - return ret - - -@_deprecation_message -def load_balancer_present( - name, - resource_group, - sku=None, - frontend_ip_configurations=None, - backend_address_pools=None, - load_balancing_rules=None, - probes=None, - inbound_nat_rules=None, - inbound_nat_pools=None, - outbound_nat_rules=None, - tags=None, - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure a load balancer exists. - - :param name: - Name of the load balancer. - - :param resource_group: - The resource group assigned to the load balancer. - - :param sku: - The load balancer SKU, which can be 'Basic' or 'Standard'. - - :param tags: - A dictionary of strings can be passed as tag metadata to the load balancer object. - - :param frontend_ip_configurations: - An optional list of dictionaries representing valid FrontendIPConfiguration objects. A frontend IP - configuration can be either private (using private IP address and subnet parameters) or public (using a - reference to a public IP address object). Valid parameters are: - - - ``name``: The name of the resource that is unique within a resource group. - - ``private_ip_address``: The private IP address of the IP configuration. Required if - 'private_ip_allocation_method' is 'Static'. - - ``private_ip_allocation_method``: The Private IP allocation method. Possible values are: 'Static' and - 'Dynamic'. - - ``subnet``: Name of an existing subnet inside of which the frontend IP will reside. - - ``public_ip_address``: Name of an existing public IP address which will be assigned to the frontend IP object. - - :param backend_address_pools: - An optional list of dictionaries representing valid BackendAddressPool objects. Only the 'name' parameter is - valid for a BackendAddressPool dictionary. All other parameters are read-only references from other objects - linking to the backend address pool. Inbound traffic is randomly load balanced across IPs in the backend IPs. - - :param probes: - An optional list of dictionaries representing valid Probe objects. Valid parameters are: - - - ``name``: The name of the resource that is unique within a resource group. - - ``protocol``: The protocol of the endpoint. Possible values are 'Http' or 'Tcp'. If 'Tcp' is specified, a - received ACK is required for the probe to be successful. If 'Http' is specified, a 200 OK response from the - specified URI is required for the probe to be successful. - - ``port``: The port for communicating the probe. Possible values range from 1 to 65535, inclusive. - - ``interval_in_seconds``: The interval, in seconds, for how frequently to probe the endpoint for health status. - Typically, the interval is slightly less than half the allocated timeout period (in seconds) which allows two - full probes before taking the instance out of rotation. The default value is 15, the minimum value is 5. - - ``number_of_probes``: The number of probes where if no response, will result in stopping further traffic from - being delivered to the endpoint. This values allows endpoints to be taken out of rotation faster or slower - than the typical times used in Azure. - - ``request_path``: The URI used for requesting health status from the VM. Path is required if a protocol is - set to 'Http'. Otherwise, it is not allowed. There is no default value. - - :param load_balancing_rules: - An optional list of dictionaries representing valid LoadBalancingRule objects. Valid parameters are: - - - ``name``: The name of the resource that is unique within a resource group. - - ``load_distribution``: The load distribution policy for this rule. Possible values are 'Default', 'SourceIP', - and 'SourceIPProtocol'. - - ``frontend_port``: The port for the external endpoint. Port numbers for each rule must be unique within the - Load Balancer. Acceptable values are between 0 and 65534. Note that value 0 enables 'Any Port'. - - ``backend_port``: The port used for internal connections on the endpoint. Acceptable values are between 0 and - 65535. Note that value 0 enables 'Any Port'. - - ``idle_timeout_in_minutes``: The timeout for the TCP idle connection. The value can be set between 4 and 30 - minutes. The default value is 4 minutes. This element is only used when the protocol is set to TCP. - - ``enable_floating_ip``: Configures a virtual machine's endpoint for the floating IP capability required - to configure a SQL AlwaysOn Availability Group. This setting is required when using the SQL AlwaysOn - Availability Groups in SQL server. This setting can't be changed after you create the endpoint. - - ``disable_outbound_snat``: Configures SNAT for the VMs in the backend pool to use the public IP address - specified in the frontend of the load balancing rule. - - ``frontend_ip_configuration``: Name of the frontend IP configuration object used by the load balancing rule - object. - - ``backend_address_pool``: Name of the backend address pool object used by the load balancing rule object. - Inbound traffic is randomly load balanced across IPs in the backend IPs. - - ``probe``: Name of the probe object used by the load balancing rule object. - - :param inbound_nat_rules: - An optional list of dictionaries representing valid InboundNatRule objects. Defining inbound NAT rules on your - load balancer is mutually exclusive with defining an inbound NAT pool. Inbound NAT pools are referenced from - virtual machine scale sets. NICs that are associated with individual virtual machines cannot reference an - Inbound NAT pool. They have to reference individual inbound NAT rules. Valid parameters are: - - - ``name``: The name of the resource that is unique within a resource group. - - ``frontend_ip_configuration``: Name of the frontend IP configuration object used by the inbound NAT rule - object. - - ``protocol``: Possible values include 'Udp', 'Tcp', or 'All'. - - ``frontend_port``: The port for the external endpoint. Port numbers for each rule must be unique within the - Load Balancer. Acceptable values range from 1 to 65534. - - ``backend_port``: The port used for the internal endpoint. Acceptable values range from 1 to 65535. - - ``idle_timeout_in_minutes``: The timeout for the TCP idle connection. The value can be set between 4 and 30 - minutes. The default value is 4 minutes. This element is only used when the protocol is set to TCP. - - ``enable_floating_ip``: Configures a virtual machine's endpoint for the floating IP capability required - to configure a SQL AlwaysOn Availability Group. This setting is required when using the SQL AlwaysOn - Availability Groups in SQL server. This setting can't be changed after you create the endpoint. - - :param inbound_nat_pools: - An optional list of dictionaries representing valid InboundNatPool objects. They define an external port range - for inbound NAT to a single backend port on NICs associated with a load balancer. Inbound NAT rules are created - automatically for each NIC associated with the Load Balancer using an external port from this range. Defining an - Inbound NAT pool on your Load Balancer is mutually exclusive with defining inbound NAT rules. Inbound NAT pools - are referenced from virtual machine scale sets. NICs that are associated with individual virtual machines cannot - reference an inbound NAT pool. They have to reference individual inbound NAT rules. Valid parameters are: - - - ``name``: The name of the resource that is unique within a resource group. - - ``frontend_ip_configuration``: Name of the frontend IP configuration object used by the inbound NAT pool - object. - - ``protocol``: Possible values include 'Udp', 'Tcp', or 'All'. - - ``frontend_port_range_start``: The first port number in the range of external ports that will be used to - provide Inbound NAT to NICs associated with a load balancer. Acceptable values range between 1 and 65534. - - ``frontend_port_range_end``: The last port number in the range of external ports that will be used to - provide Inbound NAT to NICs associated with a load balancer. Acceptable values range between 1 and 65535. - - ``backend_port``: The port used for internal connections to the endpoint. Acceptable values are between 1 and - 65535. - - :param outbound_nat_rules: - An optional list of dictionaries representing valid OutboundNatRule objects. Valid parameters are: - - - ``name``: The name of the resource that is unique within a resource group. - - ``frontend_ip_configuration``: Name of the frontend IP configuration object used by the outbound NAT rule - object. - - ``backend_address_pool``: Name of the backend address pool object used by the outbound NAT rule object. - Outbound traffic is randomly load balanced across IPs in the backend IPs. - - ``allocated_outbound_ports``: The number of outbound ports to be used for NAT. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure load balancer exists: - azurearm_network.load_balancer_present: - - name: lb1 - - resource_group: group1 - - location: eastus - - frontend_ip_configurations: - - name: lb1_feip1 - public_ip_address: pub_ip1 - - backend_address_pools: - - name: lb1_bepool1 - - probes: - - name: lb1_webprobe1 - protocol: tcp - port: 80 - interval_in_seconds: 5 - number_of_probes: 2 - - load_balancing_rules: - - name: lb1_webprobe1 - protocol: tcp - frontend_port: 80 - backend_port: 80 - idle_timeout_in_minutes: 4 - frontend_ip_configuration: lb1_feip1 - backend_address_pool: lb1_bepool1 - probe: lb1_webprobe1 - - tags: - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - require: - - azurearm_resource: Ensure resource group exists - - azurearm_network: Ensure public IP exists - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - if sku: - sku = {"name": sku.capitalize()} - - load_bal = __salt__["azurearm_network.load_balancer_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" not in load_bal: - # tag changes - tag_changes = __utils__["dictdiffer.deep_diff"]( - load_bal.get("tags", {}), tags or {} - ) - if tag_changes: - ret["changes"]["tags"] = tag_changes - - # sku changes - if sku: - sku_changes = __utils__["dictdiffer.deep_diff"]( - load_bal.get("sku", {}), sku - ) - if sku_changes: - ret["changes"]["sku"] = sku_changes - - # frontend_ip_configurations changes - if frontend_ip_configurations: - comp_ret = __utils__["azurearm.compare_list_of_dicts"]( - load_bal.get("frontend_ip_configurations", []), - frontend_ip_configurations, - ["public_ip_address", "subnet"], - ) - - if comp_ret.get("comment"): - ret["comment"] = '"frontend_ip_configurations" {}'.format( - comp_ret["comment"] - ) - return ret - - if comp_ret.get("changes"): - ret["changes"]["frontend_ip_configurations"] = comp_ret["changes"] - - # backend_address_pools changes - if backend_address_pools: - comp_ret = __utils__["azurearm.compare_list_of_dicts"]( - load_bal.get("backend_address_pools", []), backend_address_pools - ) - - if comp_ret.get("comment"): - ret["comment"] = '"backend_address_pools" {}'.format( - comp_ret["comment"] - ) - return ret - - if comp_ret.get("changes"): - ret["changes"]["backend_address_pools"] = comp_ret["changes"] - - # probes changes - if probes: - comp_ret = __utils__["azurearm.compare_list_of_dicts"]( - load_bal.get("probes", []), probes - ) - - if comp_ret.get("comment"): - ret["comment"] = '"probes" {}'.format(comp_ret["comment"]) - return ret - - if comp_ret.get("changes"): - ret["changes"]["probes"] = comp_ret["changes"] - - # load_balancing_rules changes - if load_balancing_rules: - comp_ret = __utils__["azurearm.compare_list_of_dicts"]( - load_bal.get("load_balancing_rules", []), - load_balancing_rules, - ["frontend_ip_configuration", "backend_address_pool", "probe"], - ) - - if comp_ret.get("comment"): - ret["comment"] = '"load_balancing_rules" {}'.format(comp_ret["comment"]) - return ret - - if comp_ret.get("changes"): - ret["changes"]["load_balancing_rules"] = comp_ret["changes"] - - # inbound_nat_rules changes - if inbound_nat_rules: - comp_ret = __utils__["azurearm.compare_list_of_dicts"]( - load_bal.get("inbound_nat_rules", []), - inbound_nat_rules, - ["frontend_ip_configuration"], - ) - - if comp_ret.get("comment"): - ret["comment"] = '"inbound_nat_rules" {}'.format(comp_ret["comment"]) - return ret - - if comp_ret.get("changes"): - ret["changes"]["inbound_nat_rules"] = comp_ret["changes"] - - # inbound_nat_pools changes - if inbound_nat_pools: - comp_ret = __utils__["azurearm.compare_list_of_dicts"]( - load_bal.get("inbound_nat_pools", []), - inbound_nat_pools, - ["frontend_ip_configuration"], - ) - - if comp_ret.get("comment"): - ret["comment"] = '"inbound_nat_pools" {}'.format(comp_ret["comment"]) - return ret - - if comp_ret.get("changes"): - ret["changes"]["inbound_nat_pools"] = comp_ret["changes"] - - # outbound_nat_rules changes - if outbound_nat_rules: - comp_ret = __utils__["azurearm.compare_list_of_dicts"]( - load_bal.get("outbound_nat_rules", []), - outbound_nat_rules, - ["frontend_ip_configuration"], - ) - - if comp_ret.get("comment"): - ret["comment"] = '"outbound_nat_rules" {}'.format(comp_ret["comment"]) - return ret - - if comp_ret.get("changes"): - ret["changes"]["outbound_nat_rules"] = comp_ret["changes"] - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Load balancer {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["result"] = None - ret["comment"] = "Load balancer {} would be updated.".format(name) - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "sku": sku, - "tags": tags, - "frontend_ip_configurations": frontend_ip_configurations, - "backend_address_pools": backend_address_pools, - "load_balancing_rules": load_balancing_rules, - "probes": probes, - "inbound_nat_rules": inbound_nat_rules, - "inbound_nat_pools": inbound_nat_pools, - "outbound_nat_rules": outbound_nat_rules, - }, - } - - if __opts__["test"]: - ret["comment"] = "Load balancer {} would be created.".format(name) - ret["result"] = None - return ret - - lb_kwargs = kwargs.copy() - lb_kwargs.update(connection_auth) - - load_bal = __salt__["azurearm_network.load_balancer_create_or_update"]( - name=name, - resource_group=resource_group, - sku=sku, - tags=tags, - frontend_ip_configurations=frontend_ip_configurations, - backend_address_pools=backend_address_pools, - load_balancing_rules=load_balancing_rules, - probes=probes, - inbound_nat_rules=inbound_nat_rules, - inbound_nat_pools=inbound_nat_pools, - outbound_nat_rules=outbound_nat_rules, - **lb_kwargs - ) - - if "error" not in load_bal: - ret["result"] = True - ret["comment"] = "Load balancer {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create load balancer {}! ({})".format( - name, load_bal.get("error") - ) - return ret - - -@_deprecation_message -def load_balancer_absent(name, resource_group, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure a load balancer does not exist in the resource group. - - :param name: - Name of the load balancer. - - :param resource_group: - The resource group assigned to the load balancer. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - load_bal = __salt__["azurearm_network.load_balancer_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" in load_bal: - ret["result"] = True - ret["comment"] = "Load balancer {} was not found.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "Load balancer {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": load_bal, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_network.load_balancer_delete"]( - name, resource_group, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Load balancer {} has been deleted.".format(name) - ret["changes"] = {"old": load_bal, "new": {}} - return ret - - ret["comment"] = "Failed to delete load balancer {}!".format(name) - return ret - - -@_deprecation_message -def public_ip_address_present( - name, - resource_group, - tags=None, - sku=None, - public_ip_allocation_method=None, - public_ip_address_version=None, - dns_settings=None, - idle_timeout_in_minutes=None, - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure a public IP address exists. - - :param name: - Name of the public IP address. - - :param resource_group: - The resource group assigned to the public IP address. - - :param dns_settings: - An optional dictionary representing a valid PublicIPAddressDnsSettings object. Parameters include - 'domain_name_label' and 'reverse_fqdn', which accept strings. The 'domain_name_label' parameter is concatenated - with the regionalized DNS zone make up the fully qualified domain name associated with the public IP address. - If a domain name label is specified, an A DNS record is created for the public IP in the Microsoft Azure DNS - system. The 'reverse_fqdn' parameter is a user-visible, fully qualified domain name that resolves to this public - IP address. If the reverse FQDN is specified, then a PTR DNS record is created pointing from the IP address in - the in-addr.arpa domain to the reverse FQDN. - - :param sku: - The public IP address SKU, which can be 'Basic' or 'Standard'. - - :param public_ip_allocation_method: - The public IP allocation method. Possible values are: 'Static' and 'Dynamic'. - - :param public_ip_address_version: - The public IP address version. Possible values are: 'IPv4' and 'IPv6'. - - :param idle_timeout_in_minutes: - An integer representing the idle timeout of the public IP address. - - :param tags: - A dictionary of strings can be passed as tag metadata to the public IP address object. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure public IP exists: - azurearm_network.public_ip_address_present: - - name: pub_ip1 - - resource_group: group1 - - dns_settings: - domain_name_label: decisionlab-ext-test-label - - sku: basic - - public_ip_allocation_method: static - - public_ip_address_version: ipv4 - - idle_timeout_in_minutes: 4 - - tags: - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - require: - - azurearm_resource: Ensure resource group exists - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - if sku: - sku = {"name": sku.capitalize()} - - pub_ip = __salt__["azurearm_network.public_ip_address_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" not in pub_ip: - # tag changes - tag_changes = __utils__["dictdiffer.deep_diff"]( - pub_ip.get("tags", {}), tags or {} - ) - if tag_changes: - ret["changes"]["tags"] = tag_changes - - # dns_settings changes - if dns_settings: - if not isinstance(dns_settings, dict): - ret["comment"] = "DNS settings must be provided as a dictionary!" - return ret - - for key in dns_settings: - if dns_settings[key] != pub_ip.get("dns_settings", {}).get(key): - ret["changes"]["dns_settings"] = { - "old": pub_ip.get("dns_settings"), - "new": dns_settings, - } - break - - # sku changes - if sku: - sku_changes = __utils__["dictdiffer.deep_diff"](pub_ip.get("sku", {}), sku) - if sku_changes: - ret["changes"]["sku"] = sku_changes - - # public_ip_allocation_method changes - if public_ip_allocation_method: - if public_ip_allocation_method.capitalize() != pub_ip.get( - "public_ip_allocation_method" - ): - ret["changes"]["public_ip_allocation_method"] = { - "old": pub_ip.get("public_ip_allocation_method"), - "new": public_ip_allocation_method, - } - - # public_ip_address_version changes - if public_ip_address_version: - if ( - public_ip_address_version.lower() - != pub_ip.get("public_ip_address_version", "").lower() - ): - ret["changes"]["public_ip_address_version"] = { - "old": pub_ip.get("public_ip_address_version"), - "new": public_ip_address_version, - } - - # idle_timeout_in_minutes changes - if idle_timeout_in_minutes and ( - int(idle_timeout_in_minutes) != pub_ip.get("idle_timeout_in_minutes") - ): - ret["changes"]["idle_timeout_in_minutes"] = { - "old": pub_ip.get("idle_timeout_in_minutes"), - "new": idle_timeout_in_minutes, - } - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Public IP address {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["result"] = None - ret["comment"] = "Public IP address {} would be updated.".format(name) - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "tags": tags, - "dns_settings": dns_settings, - "sku": sku, - "public_ip_allocation_method": public_ip_allocation_method, - "public_ip_address_version": public_ip_address_version, - "idle_timeout_in_minutes": idle_timeout_in_minutes, - }, - } - - if __opts__["test"]: - ret["comment"] = "Public IP address {} would be created.".format(name) - ret["result"] = None - return ret - - pub_ip_kwargs = kwargs.copy() - pub_ip_kwargs.update(connection_auth) - - pub_ip = __salt__["azurearm_network.public_ip_address_create_or_update"]( - name=name, - resource_group=resource_group, - sku=sku, - tags=tags, - dns_settings=dns_settings, - public_ip_allocation_method=public_ip_allocation_method, - public_ip_address_version=public_ip_address_version, - idle_timeout_in_minutes=idle_timeout_in_minutes, - **pub_ip_kwargs - ) - - if "error" not in pub_ip: - ret["result"] = True - ret["comment"] = "Public IP address {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create public IP address {}! ({})".format( - name, pub_ip.get("error") - ) - return ret - - -@_deprecation_message -def public_ip_address_absent(name, resource_group, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure a public IP address does not exist in the resource group. - - :param name: - Name of the public IP address. - - :param resource_group: - The resource group assigned to the public IP address. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - pub_ip = __salt__["azurearm_network.public_ip_address_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" in pub_ip: - ret["result"] = True - ret["comment"] = "Public IP address {} was not found.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "Public IP address {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": pub_ip, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_network.public_ip_address_delete"]( - name, resource_group, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Public IP address {} has been deleted.".format(name) - ret["changes"] = {"old": pub_ip, "new": {}} - return ret - - ret["comment"] = "Failed to delete public IP address {}!".format(name) - return ret - - -@_deprecation_message -def network_interface_present( - name, - ip_configurations, - subnet, - virtual_network, - resource_group, - tags=None, - virtual_machine=None, - network_security_group=None, - dns_settings=None, - mac_address=None, - primary=None, - enable_accelerated_networking=None, - enable_ip_forwarding=None, - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure a network interface exists. - - :param name: - Name of the network interface. - - :param ip_configurations: - A list of dictionaries representing valid NetworkInterfaceIPConfiguration objects. The 'name' key is required at - minimum. At least one IP Configuration must be present. - - :param subnet: - Name of the existing subnet assigned to the network interface. - - :param virtual_network: - Name of the existing virtual network containing the subnet. - - :param resource_group: - The resource group assigned to the virtual network. - - :param tags: - A dictionary of strings can be passed as tag metadata to the network interface object. - - :param network_security_group: - The name of the existing network security group to assign to the network interface. - - :param virtual_machine: - The name of the existing virtual machine to assign to the network interface. - - :param dns_settings: - An optional dictionary representing a valid NetworkInterfaceDnsSettings object. Valid parameters are: - - - ``dns_servers``: List of DNS server IP addresses. Use 'AzureProvidedDNS' to switch to Azure provided DNS - resolution. 'AzureProvidedDNS' value cannot be combined with other IPs, it must be the only value in - dns_servers collection. - - ``internal_dns_name_label``: Relative DNS name for this NIC used for internal communications between VMs in - the same virtual network. - - ``internal_fqdn``: Fully qualified DNS name supporting internal communications between VMs in the same virtual - network. - - ``internal_domain_name_suffix``: Even if internal_dns_name_label is not specified, a DNS entry is created for - the primary NIC of the VM. This DNS name can be constructed by concatenating the VM name with the value of - internal_domain_name_suffix. - - :param mac_address: - Optional string containing the MAC address of the network interface. - - :param primary: - Optional boolean allowing the interface to be set as the primary network interface on a virtual machine - with multiple interfaces attached. - - :param enable_accelerated_networking: - Optional boolean indicating whether accelerated networking should be enabled for the interface. - - :param enable_ip_forwarding: - Optional boolean indicating whether IP forwarding should be enabled for the interface. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure network interface exists: - azurearm_network.network_interface_present: - - name: iface1 - - subnet: vnet1_sn1 - - virtual_network: vnet1 - - resource_group: group1 - - ip_configurations: - - name: iface1_ipc1 - public_ip_address: pub_ip2 - - dns_settings: - internal_dns_name_label: decisionlab-int-test-label - - primary: True - - enable_accelerated_networking: True - - enable_ip_forwarding: False - - network_security_group: nsg1 - - connection_auth: {{ profile }} - - require: - - azurearm_network: Ensure subnet exists - - azurearm_network: Ensure network security group exists - - azurearm_network: Ensure another public IP exists - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - iface = __salt__["azurearm_network.network_interface_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" not in iface: - # tag changes - tag_changes = __utils__["dictdiffer.deep_diff"]( - iface.get("tags", {}), tags or {} - ) - if tag_changes: - ret["changes"]["tags"] = tag_changes - - # mac_address changes - if mac_address and (mac_address != iface.get("mac_address")): - ret["changes"]["mac_address"] = { - "old": iface.get("mac_address"), - "new": mac_address, - } - - # primary changes - if primary is not None: - if primary != iface.get("primary", True): - ret["changes"]["primary"] = { - "old": iface.get("primary"), - "new": primary, - } - - # enable_accelerated_networking changes - if enable_accelerated_networking is not None: - if enable_accelerated_networking != iface.get( - "enable_accelerated_networking" - ): - ret["changes"]["enable_accelerated_networking"] = { - "old": iface.get("enable_accelerated_networking"), - "new": enable_accelerated_networking, - } - - # enable_ip_forwarding changes - if enable_ip_forwarding is not None: - if enable_ip_forwarding != iface.get("enable_ip_forwarding"): - ret["changes"]["enable_ip_forwarding"] = { - "old": iface.get("enable_ip_forwarding"), - "new": enable_ip_forwarding, - } - - # network_security_group changes - nsg_name = None - if iface.get("network_security_group"): - nsg_name = iface["network_security_group"]["id"].split("/")[-1] - - if network_security_group and (network_security_group != nsg_name): - ret["changes"]["network_security_group"] = { - "old": nsg_name, - "new": network_security_group, - } - - # virtual_machine changes - vm_name = None - if iface.get("virtual_machine"): - vm_name = iface["virtual_machine"]["id"].split("/")[-1] - - if virtual_machine and (virtual_machine != vm_name): - ret["changes"]["virtual_machine"] = {"old": vm_name, "new": virtual_machine} - - # dns_settings changes - if dns_settings: - if not isinstance(dns_settings, dict): - ret["comment"] = "DNS settings must be provided as a dictionary!" - return ret - - for key in dns_settings: - if ( - dns_settings[key].lower() - != iface.get("dns_settings", {}).get(key, "").lower() - ): - ret["changes"]["dns_settings"] = { - "old": iface.get("dns_settings"), - "new": dns_settings, - } - break - - # ip_configurations changes - comp_ret = __utils__["azurearm.compare_list_of_dicts"]( - iface.get("ip_configurations", []), - ip_configurations, - ["public_ip_address", "subnet"], - ) - - if comp_ret.get("comment"): - ret["comment"] = '"ip_configurations" {}'.format(comp_ret["comment"]) - return ret - - if comp_ret.get("changes"): - ret["changes"]["ip_configurations"] = comp_ret["changes"] - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Network interface {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["result"] = None - ret["comment"] = "Network interface {} would be updated.".format(name) - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "ip_configurations": ip_configurations, - "dns_settings": dns_settings, - "network_security_group": network_security_group, - "virtual_machine": virtual_machine, - "enable_accelerated_networking": enable_accelerated_networking, - "enable_ip_forwarding": enable_ip_forwarding, - "mac_address": mac_address, - "primary": primary, - "tags": tags, - }, - } - - if __opts__["test"]: - ret["comment"] = "Network interface {} would be created.".format(name) - ret["result"] = None - return ret - - iface_kwargs = kwargs.copy() - iface_kwargs.update(connection_auth) - - iface = __salt__["azurearm_network.network_interface_create_or_update"]( - name=name, - subnet=subnet, - virtual_network=virtual_network, - resource_group=resource_group, - ip_configurations=ip_configurations, - dns_settings=dns_settings, - enable_accelerated_networking=enable_accelerated_networking, - enable_ip_forwarding=enable_ip_forwarding, - mac_address=mac_address, - primary=primary, - network_security_group=network_security_group, - virtual_machine=virtual_machine, - tags=tags, - **iface_kwargs - ) - - if "error" not in iface: - ret["result"] = True - ret["comment"] = "Network interface {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create network interface {}! ({})".format( - name, iface.get("error") - ) - return ret - - -@_deprecation_message -def network_interface_absent(name, resource_group, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure a network interface does not exist in the resource group. - - :param name: - Name of the network interface. - - :param resource_group: - The resource group assigned to the network interface. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - iface = __salt__["azurearm_network.network_interface_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" in iface: - ret["result"] = True - ret["comment"] = "Network interface {} was not found.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "Network interface {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": iface, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_network.network_interface_delete"]( - name, resource_group, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Network interface {} has been deleted.".format(name) - ret["changes"] = {"old": iface, "new": {}} - return ret - - ret["comment"] = "Failed to delete network interface {}!)".format(name) - return ret - - -@_deprecation_message -def route_table_present( - name, - resource_group, - tags=None, - routes=None, - disable_bgp_route_propagation=None, - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure a route table exists. - - :param name: - Name of the route table. - - :param resource_group: - The resource group assigned to the route table. - - :param routes: - An optional list of dictionaries representing valid Route objects contained within a route table. See the - documentation for the route_present state or route_create_or_update execution module for more information on - required and optional parameters for routes. The routes are only managed if this parameter is present. When this - parameter is absent, implemented routes will not be removed, and will merely become unmanaged. - - :param disable_bgp_route_propagation: - An optional boolean parameter setting whether to disable the routes learned by BGP on the route table. - - :param tags: - A dictionary of strings can be passed as tag metadata to the route table object. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure route table exists: - azurearm_network.route_table_present: - - name: rt1 - - resource_group: group1 - - routes: - - name: rt1_route1 - address_prefix: '0.0.0.0/0' - next_hop_type: internet - - name: rt1_route2 - address_prefix: '192.168.0.0/16' - next_hop_type: vnetlocal - - tags: - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - require: - - azurearm_resource: Ensure resource group exists - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - rt_tbl = __salt__["azurearm_network.route_table_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" not in rt_tbl: - # tag changes - tag_changes = __utils__["dictdiffer.deep_diff"]( - rt_tbl.get("tags", {}), tags or {} - ) - if tag_changes: - ret["changes"]["tags"] = tag_changes - - # disable_bgp_route_propagation changes - # pylint: disable=line-too-long - if disable_bgp_route_propagation and ( - disable_bgp_route_propagation != rt_tbl.get("disable_bgp_route_propagation") - ): - ret["changes"]["disable_bgp_route_propagation"] = { - "old": rt_tbl.get("disable_bgp_route_propagation"), - "new": disable_bgp_route_propagation, - } - - # routes changes - if routes: - comp_ret = __utils__["azurearm.compare_list_of_dicts"]( - rt_tbl.get("routes", []), routes - ) - - if comp_ret.get("comment"): - ret["comment"] = '"routes" {}'.format(comp_ret["comment"]) - return ret - - if comp_ret.get("changes"): - ret["changes"]["routes"] = comp_ret["changes"] - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Route table {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["result"] = None - ret["comment"] = "Route table {} would be updated.".format(name) - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "tags": tags, - "routes": routes, - "disable_bgp_route_propagation": disable_bgp_route_propagation, - }, - } - - if __opts__["test"]: - ret["comment"] = "Route table {} would be created.".format(name) - ret["result"] = None - return ret - - rt_tbl_kwargs = kwargs.copy() - rt_tbl_kwargs.update(connection_auth) - - rt_tbl = __salt__["azurearm_network.route_table_create_or_update"]( - name=name, - resource_group=resource_group, - disable_bgp_route_propagation=disable_bgp_route_propagation, - routes=routes, - tags=tags, - **rt_tbl_kwargs - ) - - if "error" not in rt_tbl: - ret["result"] = True - ret["comment"] = "Route table {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create route table {}! ({})".format( - name, rt_tbl.get("error") - ) - return ret - - -@_deprecation_message -def route_table_absent(name, resource_group, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure a route table does not exist in the resource group. - - :param name: - Name of the route table. - - :param resource_group: - The resource group assigned to the route table. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - rt_tbl = __salt__["azurearm_network.route_table_get"]( - name, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" in rt_tbl: - ret["result"] = True - ret["comment"] = "Route table {} was not found.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "Route table {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": rt_tbl, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_network.route_table_delete"]( - name, resource_group, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Route table {} has been deleted.".format(name) - ret["changes"] = {"old": rt_tbl, "new": {}} - return ret - - ret["comment"] = "Failed to delete route table {}!".format(name) - return ret - - -@_deprecation_message -def route_present( - name, - address_prefix, - next_hop_type, - route_table, - resource_group, - next_hop_ip_address=None, - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure a route exists within a route table. - - :param name: - Name of the route. - - :param address_prefix: - The destination CIDR to which the route applies. - - :param next_hop_type: - The type of Azure hop the packet should be sent to. Possible values are: 'VirtualNetworkGateway', 'VnetLocal', - 'Internet', 'VirtualAppliance', and 'None'. - - :param next_hop_ip_address: - The IP address packets should be forwarded to. Next hop values are only allowed in routes where the next hop - type is 'VirtualAppliance'. - - :param route_table: - The name of the existing route table which will contain the route. - - :param resource_group: - The resource group assigned to the route table. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure route exists: - azurearm_network.route_present: - - name: rt1_route2 - - route_table: rt1 - - resource_group: group1 - - address_prefix: '192.168.0.0/16' - - next_hop_type: vnetlocal - - connection_auth: {{ profile }} - - require: - - azurearm_network: Ensure route table exists - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - route = __salt__["azurearm_network.route_get"]( - name, route_table, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" not in route: - if address_prefix != route.get("address_prefix"): - ret["changes"]["address_prefix"] = { - "old": route.get("address_prefix"), - "new": address_prefix, - } - - if next_hop_type.lower() != route.get("next_hop_type", "").lower(): - ret["changes"]["next_hop_type"] = { - "old": route.get("next_hop_type"), - "new": next_hop_type, - } - - if ( - next_hop_type.lower() == "virtualappliance" - and next_hop_ip_address != route.get("next_hop_ip_address") - ): - ret["changes"]["next_hop_ip_address"] = { - "old": route.get("next_hop_ip_address"), - "new": next_hop_ip_address, - } - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Route {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["result"] = None - ret["comment"] = "Route {} would be updated.".format(name) - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "address_prefix": address_prefix, - "next_hop_type": next_hop_type, - "next_hop_ip_address": next_hop_ip_address, - }, - } - - if __opts__["test"]: - ret["comment"] = "Route {} would be created.".format(name) - ret["result"] = None - return ret - - route_kwargs = kwargs.copy() - route_kwargs.update(connection_auth) - - route = __salt__["azurearm_network.route_create_or_update"]( - name=name, - route_table=route_table, - resource_group=resource_group, - address_prefix=address_prefix, - next_hop_type=next_hop_type, - next_hop_ip_address=next_hop_ip_address, - **route_kwargs - ) - - if "error" not in route: - ret["result"] = True - ret["comment"] = "Route {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create route {}! ({})".format(name, route.get("error")) - return ret - - -@_deprecation_message -def route_absent(name, route_table, resource_group, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure a route table does not exist in the resource group. - - :param name: - Name of the route table. - - :param route_table: - The name of the existing route table containing the route. - - :param resource_group: - The resource group assigned to the route table. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - route = __salt__["azurearm_network.route_get"]( - name, route_table, resource_group, azurearm_log_level="info", **connection_auth - ) - - if "error" in route: - ret["result"] = True - ret["comment"] = "Route {} was not found.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "Route {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": route, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_network.route_delete"]( - name, route_table, resource_group, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Route {} has been deleted.".format(name) - ret["changes"] = {"old": route, "new": {}} - return ret - - ret["comment"] = "Failed to delete route {}!".format(name) - return ret diff --git a/salt/states/azurearm_resource.py b/salt/states/azurearm_resource.py deleted file mode 100644 index 7cef3bc0bd9..00000000000 --- a/salt/states/azurearm_resource.py +++ /dev/null @@ -1,880 +0,0 @@ -""" -Azure (ARM) Resource State Module - -.. versionadded:: 2019.2.0 - -.. warning:: - - This cloud provider will be removed from Salt in version 3007 in favor of - the `saltext.azurerm Salt Extension - `_ - -:maintainer: -:maturity: new -:depends: - * `azure `_ >= 2.0.0 - * `azure-common `_ >= 1.1.8 - * `azure-mgmt `_ >= 1.0.0 - * `azure-mgmt-compute `_ >= 1.0.0 - * `azure-mgmt-network `_ >= 1.7.1 - * `azure-mgmt-resource `_ >= 1.1.0 - * `azure-mgmt-storage `_ >= 1.0.0 - * `azure-mgmt-web `_ >= 0.32.0 - * `azure-storage `_ >= 0.34.3 - * `msrestazure `_ >= 0.4.21 -:platform: linux - -:configuration: This module requires Azure Resource Manager credentials to be passed as a dictionary of - keyword arguments to the ``connection_auth`` parameter in order to work properly. Since the authentication - parameters are sensitive, it's recommended to pass them to the states via pillar. - - Required provider parameters: - - if using username and password: - * ``subscription_id`` - * ``username`` - * ``password`` - - if using a service principal: - * ``subscription_id`` - * ``tenant`` - * ``client_id`` - * ``secret`` - - Optional provider parameters: - - **cloud_environment**: Used to point the cloud driver to different API endpoints, such as Azure GovCloud. Possible values: - * ``AZURE_PUBLIC_CLOUD`` (default) - * ``AZURE_CHINA_CLOUD`` - * ``AZURE_US_GOV_CLOUD`` - * ``AZURE_GERMAN_CLOUD`` - - Example Pillar for Azure Resource Manager authentication: - - .. code-block:: yaml - - azurearm: - user_pass_auth: - subscription_id: 3287abc8-f98a-c678-3bde-326766fd3617 - username: fletch - password: 123pass - mysubscription: - subscription_id: 3287abc8-f98a-c678-3bde-326766fd3617 - tenant: ABCDEFAB-1234-ABCD-1234-ABCDEFABCDEF - client_id: ABCDEFAB-1234-ABCD-1234-ABCDEFABCDEF - secret: XXXXXXXXXXXXXXXXXXXXXXXX - cloud_environment: AZURE_PUBLIC_CLOUD - - Example states using Azure Resource Manager authentication: - - .. code-block:: jinja - - {% set profile = salt['pillar.get']('azurearm:mysubscription') %} - Ensure resource group exists: - azurearm_resource.resource_group_present: - - name: my_rg - - location: westus - - tags: - how_awesome: very - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - Ensure resource group is absent: - azurearm_resource.resource_group_absent: - - name: other_rg - - connection_auth: {{ profile }} - -""" - - -import json -import logging -from functools import wraps - -import salt.utils.azurearm -import salt.utils.files - -__virtualname__ = "azurearm_resource" - -log = logging.getLogger(__name__) - - -def __virtual__(): - """ - Only make this state available if the azurearm_resource module is available. - """ - if "azurearm_resource.resource_group_check_existence" in __salt__: - return __virtualname__ - return (False, "azurearm_resource module could not be loaded") - - -def _deprecation_message(function): - """ - Decorator wrapper to warn about azurearm deprecation - """ - - @wraps(function) - def wrapped(*args, **kwargs): - salt.utils.versions.warn_until( - "Chlorine", - "The 'azurearm' functionality in Salt has been deprecated and its " - "functionality will be removed in version 3007 in favor of the " - "saltext.azurerm Salt Extension. " - "(https://github.com/salt-extensions/saltext-azurerm)", - category=FutureWarning, - ) - ret = function(*args, **salt.utils.args.clean_kwargs(**kwargs)) - return ret - - return wrapped - - -@_deprecation_message -def resource_group_present( - name, location, managed_by=None, tags=None, connection_auth=None, **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure a resource group exists. - - :param name: - Name of the resource group. - - :param location: - The Azure location in which to create the resource group. This value cannot be updated once - the resource group is created. - - :param managed_by: - The ID of the resource that manages this resource group. This value cannot be updated once - the resource group is created. - - :param tags: - A dictionary of strings can be passed as tag metadata to the resource group object. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure resource group exists: - azurearm_resource.resource_group_present: - - name: group1 - - location: eastus - - tags: - contact_name: Elmer Fudd Gantry - - connection_auth: {{ profile }} - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - group = {} - - present = __salt__["azurearm_resource.resource_group_check_existence"]( - name, **connection_auth - ) - - if present: - group = __salt__["azurearm_resource.resource_group_get"]( - name, **connection_auth - ) - ret["changes"] = __utils__["dictdiffer.deep_diff"]( - group.get("tags", {}), tags or {} - ) - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Resource group {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["comment"] = "Resource group {} tags would be updated.".format(name) - ret["result"] = None - ret["changes"] = {"old": group.get("tags", {}), "new": tags} - return ret - - elif __opts__["test"]: - ret["comment"] = "Resource group {} would be created.".format(name) - ret["result"] = None - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "location": location, - "managed_by": managed_by, - "tags": tags, - }, - } - return ret - - group_kwargs = kwargs.copy() - group_kwargs.update(connection_auth) - - group = __salt__["azurearm_resource.resource_group_create_or_update"]( - name, location, managed_by=managed_by, tags=tags, **group_kwargs - ) - present = __salt__["azurearm_resource.resource_group_check_existence"]( - name, **connection_auth - ) - - if present: - ret["result"] = True - ret["comment"] = "Resource group {} has been created.".format(name) - ret["changes"] = {"old": {}, "new": group} - return ret - - ret["comment"] = "Failed to create resource group {}! ({})".format( - name, group.get("error") - ) - return ret - - -@_deprecation_message -def resource_group_absent(name, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure a resource group does not exist in the current subscription. - - :param name: - Name of the resource group. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - group = {} - - present = __salt__["azurearm_resource.resource_group_check_existence"]( - name, **connection_auth - ) - - if not present: - ret["result"] = True - ret["comment"] = "Resource group {} is already absent.".format(name) - return ret - - elif __opts__["test"]: - group = __salt__["azurearm_resource.resource_group_get"]( - name, **connection_auth - ) - - ret["comment"] = "Resource group {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": group, - "new": {}, - } - return ret - - group = __salt__["azurearm_resource.resource_group_get"](name, **connection_auth) - deleted = __salt__["azurearm_resource.resource_group_delete"]( - name, **connection_auth - ) - - if deleted: - present = False - else: - present = __salt__["azurearm_resource.resource_group_check_existence"]( - name, **connection_auth - ) - - if not present: - ret["result"] = True - ret["comment"] = "Resource group {} has been deleted.".format(name) - ret["changes"] = {"old": group, "new": {}} - return ret - - ret["comment"] = "Failed to delete resource group {}!".format(name) - return ret - - -@_deprecation_message -def policy_definition_present( - name, - policy_rule=None, - policy_type=None, - mode=None, - display_name=None, - description=None, - metadata=None, - parameters=None, - policy_rule_json=None, - policy_rule_file=None, - template="jinja", - source_hash=None, - source_hash_name=None, - skip_verify=False, - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure a security policy definition exists. - - :param name: - Name of the policy definition. - - :param policy_rule: - A YAML dictionary defining the policy rule. See `Azure Policy Definition documentation - `_ for details on the - structure. One of ``policy_rule``, ``policy_rule_json``, or ``policy_rule_file`` is required, in that order of - precedence for use if multiple parameters are used. - - :param policy_rule_json: - A text field defining the entirety of a policy definition in JSON. See `Azure Policy Definition documentation - `_ for details on the - structure. One of ``policy_rule``, ``policy_rule_json``, or ``policy_rule_file`` is required, in that order of - precedence for use if multiple parameters are used. Note that the `name` field in the JSON will override the - ``name`` parameter in the state. - - :param policy_rule_file: - The source of a JSON file defining the entirety of a policy definition. See `Azure Policy Definition - documentation `_ for - details on the structure. One of ``policy_rule``, ``policy_rule_json``, or ``policy_rule_file`` is required, - in that order of precedence for use if multiple parameters are used. Note that the `name` field in the JSON - will override the ``name`` parameter in the state. - - :param skip_verify: - Used for the ``policy_rule_file`` parameter. If ``True``, hash verification of remote file sources - (``http://``, ``https://``, ``ftp://``) will be skipped, and the ``source_hash`` argument will be ignored. - - :param source_hash: - This can be a source hash string or the URI of a file that contains source hash strings. - - :param source_hash_name: - When ``source_hash`` refers to a hash file, Salt will try to find the correct hash by matching the - filename/URI associated with that hash. - - :param policy_type: - The type of policy definition. Possible values are NotSpecified, BuiltIn, and Custom. Only used with the - ``policy_rule`` parameter. - - :param mode: - The policy definition mode. Possible values are NotSpecified, Indexed, and All. Only used with the - ``policy_rule`` parameter. - - :param display_name: - The display name of the policy definition. Only used with the ``policy_rule`` parameter. - - :param description: - The policy definition description. Only used with the ``policy_rule`` parameter. - - :param metadata: - The policy definition metadata defined as a dictionary. Only used with the ``policy_rule`` parameter. - - :param parameters: - Required dictionary if a parameter is used in the policy rule. Only used with the ``policy_rule`` parameter. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure policy definition exists: - azurearm_resource.policy_definition_present: - - name: testpolicy - - display_name: Test Policy - - description: Test policy for testing policies. - - policy_rule: - if: - allOf: - - equals: Microsoft.Compute/virtualMachines/write - source: action - - field: location - in: - - eastus - - eastus2 - - centralus - then: - effect: deny - - connection_auth: {{ profile }} - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - if not policy_rule and not policy_rule_json and not policy_rule_file: - ret["comment"] = ( - 'One of "policy_rule", "policy_rule_json", or "policy_rule_file" is' - " required!" - ) - return ret - - if ( - sum(x is not None for x in [policy_rule, policy_rule_json, policy_rule_file]) - > 1 - ): - ret["comment"] = ( - 'Only one of "policy_rule", "policy_rule_json", or "policy_rule_file" is' - " allowed!" - ) - return ret - - if (policy_rule_json or policy_rule_file) and ( - policy_type or mode or display_name or description or metadata or parameters - ): - ret["comment"] = ( - 'Policy definitions cannot be passed when "policy_rule_json" or' - ' "policy_rule_file" is defined!' - ) - return ret - - temp_rule = {} - if policy_rule_json: - try: - temp_rule = json.loads(policy_rule_json) - except Exception as exc: # pylint: disable=broad-except - ret["comment"] = "Unable to load policy rule json! ({})".format(exc) - return ret - elif policy_rule_file: - try: - # pylint: disable=unused-variable - sfn, source_sum, comment_ = __salt__["file.get_managed"]( - None, - template, - policy_rule_file, - source_hash, - source_hash_name, - None, - None, - None, - __env__, - None, - None, - skip_verify=skip_verify, - **kwargs - ) - except Exception as exc: # pylint: disable=broad-except - ret["comment"] = 'Unable to locate policy rule file "{}"! ({})'.format( - policy_rule_file, exc - ) - return ret - - if not sfn: - ret["comment"] = 'Unable to locate policy rule file "{}"!)'.format( - policy_rule_file - ) - return ret - - try: - with salt.utils.files.fopen(sfn, "r") as prf: - temp_rule = json.load(prf) - except Exception as exc: # pylint: disable=broad-except - ret["comment"] = 'Unable to load policy rule file "{}"! ({})'.format( - policy_rule_file, exc - ) - return ret - - if sfn: - salt.utils.files.remove(sfn) - - policy_name = name - if policy_rule_json or policy_rule_file: - if temp_rule.get("name"): - policy_name = temp_rule.get("name") - policy_rule = temp_rule.get("properties", {}).get("policyRule") - policy_type = temp_rule.get("properties", {}).get("policyType") - mode = temp_rule.get("properties", {}).get("mode") - display_name = temp_rule.get("properties", {}).get("displayName") - description = temp_rule.get("properties", {}).get("description") - metadata = temp_rule.get("properties", {}).get("metadata") - parameters = temp_rule.get("properties", {}).get("parameters") - - policy = __salt__["azurearm_resource.policy_definition_get"]( - name, azurearm_log_level="info", **connection_auth - ) - - if "error" not in policy: - if policy_type and policy_type.lower() != policy.get("policy_type", "").lower(): - ret["changes"]["policy_type"] = { - "old": policy.get("policy_type"), - "new": policy_type, - } - - if (mode or "").lower() != policy.get("mode", "").lower(): - ret["changes"]["mode"] = {"old": policy.get("mode"), "new": mode} - - if (display_name or "").lower() != policy.get("display_name", "").lower(): - ret["changes"]["display_name"] = { - "old": policy.get("display_name"), - "new": display_name, - } - - if (description or "").lower() != policy.get("description", "").lower(): - ret["changes"]["description"] = { - "old": policy.get("description"), - "new": description, - } - - rule_changes = __utils__["dictdiffer.deep_diff"]( - policy.get("policy_rule", {}), policy_rule or {} - ) - if rule_changes: - ret["changes"]["policy_rule"] = rule_changes - - meta_changes = __utils__["dictdiffer.deep_diff"]( - policy.get("metadata", {}), metadata or {} - ) - if meta_changes: - ret["changes"]["metadata"] = meta_changes - - param_changes = __utils__["dictdiffer.deep_diff"]( - policy.get("parameters", {}), parameters or {} - ) - if param_changes: - ret["changes"]["parameters"] = param_changes - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Policy definition {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["comment"] = "Policy definition {} would be updated.".format(name) - ret["result"] = None - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": policy_name, - "policy_type": policy_type, - "mode": mode, - "display_name": display_name, - "description": description, - "metadata": metadata, - "parameters": parameters, - "policy_rule": policy_rule, - }, - } - - if __opts__["test"]: - ret["comment"] = "Policy definition {} would be created.".format(name) - ret["result"] = None - return ret - - # Convert OrderedDict to dict - if isinstance(metadata, dict): - metadata = json.loads(json.dumps(metadata)) - if isinstance(parameters, dict): - parameters = json.loads(json.dumps(parameters)) - - policy_kwargs = kwargs.copy() - policy_kwargs.update(connection_auth) - - policy = __salt__["azurearm_resource.policy_definition_create_or_update"]( - name=policy_name, - policy_rule=policy_rule, - policy_type=policy_type, - mode=mode, - display_name=display_name, - description=description, - metadata=metadata, - parameters=parameters, - **policy_kwargs - ) - - if "error" not in policy: - ret["result"] = True - ret["comment"] = "Policy definition {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create policy definition {}! ({})".format( - name, policy.get("error") - ) - return ret - - -@_deprecation_message -def policy_definition_absent(name, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure a policy definition does not exist in the current subscription. - - :param name: - Name of the policy definition. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - policy = __salt__["azurearm_resource.policy_definition_get"]( - name, azurearm_log_level="info", **connection_auth - ) - - if "error" in policy: - ret["result"] = True - ret["comment"] = "Policy definition {} is already absent.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "Policy definition {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": policy, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_resource.policy_definition_delete"]( - name, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Policy definition {} has been deleted.".format(name) - ret["changes"] = {"old": policy, "new": {}} - return ret - - ret["comment"] = "Failed to delete policy definition {}!".format(name) - return ret - - -@_deprecation_message -def policy_assignment_present( - name, - scope, - definition_name, - display_name=None, - description=None, - assignment_type=None, - parameters=None, - connection_auth=None, - **kwargs -): - """ - .. versionadded:: 2019.2.0 - - Ensure a security policy assignment exists. - - :param name: - Name of the policy assignment. - - :param scope: - The scope of the policy assignment. - - :param definition_name: - The name of the policy definition to assign. - - :param display_name: - The display name of the policy assignment. - - :param description: - The policy assignment description. - - :param assignment_type: - The type of policy assignment. - - :param parameters: - Required dictionary if a parameter is used in the policy rule. - - :param connection_auth: - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - - Example usage: - - .. code-block:: yaml - - Ensure policy assignment exists: - azurearm_resource.policy_assignment_present: - - name: testassign - - scope: /subscriptions/bc75htn-a0fhsi-349b-56gh-4fghti-f84852 - - definition_name: testpolicy - - display_name: Test Assignment - - description: Test assignment for testing assignments. - - connection_auth: {{ profile }} - - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - policy = __salt__["azurearm_resource.policy_assignment_get"]( - name, scope, azurearm_log_level="info", **connection_auth - ) - - if "error" not in policy: - if ( - assignment_type - and assignment_type.lower() != policy.get("type", "").lower() - ): - ret["changes"]["type"] = {"old": policy.get("type"), "new": assignment_type} - - if scope.lower() != policy["scope"].lower(): - ret["changes"]["scope"] = {"old": policy["scope"], "new": scope} - - pa_name = policy["policy_definition_id"].split("/")[-1] - if definition_name.lower() != pa_name.lower(): - ret["changes"]["definition_name"] = {"old": pa_name, "new": definition_name} - - if (display_name or "").lower() != policy.get("display_name", "").lower(): - ret["changes"]["display_name"] = { - "old": policy.get("display_name"), - "new": display_name, - } - - if (description or "").lower() != policy.get("description", "").lower(): - ret["changes"]["description"] = { - "old": policy.get("description"), - "new": description, - } - - param_changes = __utils__["dictdiffer.deep_diff"]( - policy.get("parameters", {}), parameters or {} - ) - if param_changes: - ret["changes"]["parameters"] = param_changes - - if not ret["changes"]: - ret["result"] = True - ret["comment"] = "Policy assignment {} is already present.".format(name) - return ret - - if __opts__["test"]: - ret["comment"] = "Policy assignment {} would be updated.".format(name) - ret["result"] = None - return ret - - else: - ret["changes"] = { - "old": {}, - "new": { - "name": name, - "scope": scope, - "definition_name": definition_name, - "type": assignment_type, - "display_name": display_name, - "description": description, - "parameters": parameters, - }, - } - - if __opts__["test"]: - ret["comment"] = "Policy assignment {} would be created.".format(name) - ret["result"] = None - return ret - - if isinstance(parameters, dict): - parameters = json.loads(json.dumps(parameters)) - - policy_kwargs = kwargs.copy() - policy_kwargs.update(connection_auth) - policy = __salt__["azurearm_resource.policy_assignment_create"]( - name=name, - scope=scope, - definition_name=definition_name, - type=assignment_type, - display_name=display_name, - description=description, - parameters=parameters, - **policy_kwargs - ) - - if "error" not in policy: - ret["result"] = True - ret["comment"] = "Policy assignment {} has been created.".format(name) - return ret - - ret["comment"] = "Failed to create policy assignment {}! ({})".format( - name, policy.get("error") - ) - return ret - - -@_deprecation_message -def policy_assignment_absent(name, scope, connection_auth=None): - """ - .. versionadded:: 2019.2.0 - - Ensure a policy assignment does not exist in the provided scope. - - :param name: - Name of the policy assignment. - - :param scope: - The scope of the policy assignment. - - connection_auth - A dict with subscription and authentication parameters to be used in connecting to the - Azure Resource Manager API. - """ - ret = {"name": name, "result": False, "comment": "", "changes": {}} - - if not isinstance(connection_auth, dict): - ret[ - "comment" - ] = "Connection information must be specified via connection_auth dictionary!" - return ret - - policy = __salt__["azurearm_resource.policy_assignment_get"]( - name, scope, azurearm_log_level="info", **connection_auth - ) - - if "error" in policy: - ret["result"] = True - ret["comment"] = "Policy assignment {} is already absent.".format(name) - return ret - - elif __opts__["test"]: - ret["comment"] = "Policy assignment {} would be deleted.".format(name) - ret["result"] = None - ret["changes"] = { - "old": policy, - "new": {}, - } - return ret - - deleted = __salt__["azurearm_resource.policy_assignment_delete"]( - name, scope, **connection_auth - ) - - if deleted: - ret["result"] = True - ret["comment"] = "Policy assignment {} has been deleted.".format(name) - ret["changes"] = {"old": policy, "new": {}} - return ret - - ret["comment"] = "Failed to delete policy assignment {}!".format(name) - return ret diff --git a/salt/utils/azurearm.py b/salt/utils/azurearm.py deleted file mode 100644 index 276cbb66b36..00000000000 --- a/salt/utils/azurearm.py +++ /dev/null @@ -1,338 +0,0 @@ -""" -Azure (ARM) Utilities - -.. versionadded:: 2019.2.0 - -:maintainer: -:maturity: new -:depends: - * `azure `_ >= 2.0.0rc6 - * `azure-common `_ >= 1.1.4 - * `azure-mgmt `_ >= 0.30.0rc6 - * `azure-mgmt-compute `_ >= 0.33.0 - * `azure-mgmt-network `_ >= 0.30.0rc6 - * `azure-mgmt-resource `_ >= 0.30.0 - * `azure-mgmt-storage `_ >= 0.30.0rc6 - * `azure-mgmt-web `_ >= 0.30.0rc6 - * `azure-storage `_ >= 0.32.0 - * `msrestazure `_ >= 0.4.21 -:platform: linux - -""" - -import importlib -import logging -import sys -from operator import itemgetter - -import salt.config -import salt.loader -import salt.utils.args -import salt.utils.stringutils -import salt.utils.versions -import salt.version -from salt.exceptions import SaltInvocationError, SaltSystemExit - -try: - from azure.common.credentials import ( - ServicePrincipalCredentials, - UserPassCredentials, - ) - from msrestazure.azure_cloud import ( - MetadataEndpointError, - get_cloud_from_metadata_endpoint, - ) - - HAS_AZURE = True -except ImportError: - HAS_AZURE = False - -__opts__ = salt.config.minion_config("/etc/salt/minion") -__salt__ = salt.loader.minion_mods(__opts__) - -log = logging.getLogger(__name__) - - -def __virtual__(): - if not HAS_AZURE: - return False - else: - return True - - -def _determine_auth(**kwargs): - """ - Acquire Azure ARM Credentials - """ - if "profile" in kwargs: - azure_credentials = __salt__["config.option"](kwargs["profile"]) - kwargs.update(azure_credentials) - - service_principal_creds_kwargs = ["client_id", "secret", "tenant"] - user_pass_creds_kwargs = ["username", "password"] - - try: - if kwargs.get("cloud_environment") and kwargs.get( - "cloud_environment" - ).startswith("http"): - cloud_env = get_cloud_from_metadata_endpoint(kwargs["cloud_environment"]) - else: - cloud_env_module = importlib.import_module("msrestazure.azure_cloud") - cloud_env = getattr( - cloud_env_module, kwargs.get("cloud_environment", "AZURE_PUBLIC_CLOUD") - ) - except (AttributeError, ImportError, MetadataEndpointError): - raise sys.exit( - "The Azure cloud environment {} is not available.".format( - kwargs["cloud_environment"] - ) - ) - - if set(service_principal_creds_kwargs).issubset(kwargs): - if not (kwargs["client_id"] and kwargs["secret"] and kwargs["tenant"]): - raise SaltInvocationError( - "The client_id, secret, and tenant parameters must all be " - "populated if using service principals." - ) - else: - credentials = ServicePrincipalCredentials( - kwargs["client_id"], - kwargs["secret"], - tenant=kwargs["tenant"], - cloud_environment=cloud_env, - ) - elif set(user_pass_creds_kwargs).issubset(kwargs): - if not (kwargs["username"] and kwargs["password"]): - raise SaltInvocationError( - "The username and password parameters must both be " - "populated if using username/password authentication." - ) - else: - credentials = UserPassCredentials( - kwargs["username"], kwargs["password"], cloud_environment=cloud_env - ) - elif "subscription_id" in kwargs: - try: - from msrestazure.azure_active_directory import MSIAuthentication - - credentials = MSIAuthentication(cloud_environment=cloud_env) - except ImportError: - raise SaltSystemExit( - msg=( - "MSI authentication support not availabe (requires msrestazure >=" - " 0.4.14)" - ) - ) - - else: - raise SaltInvocationError( - "Unable to determine credentials. " - "A subscription_id with username and password, " - "or client_id, secret, and tenant or a profile with the " - "required parameters populated" - ) - - if "subscription_id" not in kwargs: - raise SaltInvocationError("A subscription_id must be specified") - - subscription_id = salt.utils.stringutils.to_str(kwargs["subscription_id"]) - - return credentials, subscription_id, cloud_env - - -def get_client(client_type, **kwargs): - """ - Dynamically load the selected client and return a management client object - """ - client_map = { - "compute": "ComputeManagement", - "authorization": "AuthorizationManagement", - "dns": "DnsManagement", - "storage": "StorageManagement", - "managementlock": "ManagementLock", - "monitor": "MonitorManagement", - "network": "NetworkManagement", - "policy": "Policy", - "resource": "ResourceManagement", - "subscription": "Subscription", - "web": "WebSiteManagement", - } - - if client_type not in client_map: - raise SaltSystemExit( - msg="The Azure ARM client_type {} specified can not be found.".format( - client_type - ) - ) - - map_value = client_map[client_type] - - if client_type in ["policy", "subscription"]: - module_name = "resource" - elif client_type in ["managementlock"]: - module_name = "resource.locks" - else: - module_name = client_type - - try: - client_module = importlib.import_module("azure.mgmt." + module_name) - # pylint: disable=invalid-name - Client = getattr(client_module, "{}Client".format(map_value)) - except ImportError: - raise sys.exit("The azure {} client is not available.".format(client_type)) - - credentials, subscription_id, cloud_env = _determine_auth(**kwargs) - - if client_type == "subscription": - client = Client( - credentials=credentials, - base_url=cloud_env.endpoints.resource_manager, - ) - else: - client = Client( - credentials=credentials, - subscription_id=subscription_id, - base_url=cloud_env.endpoints.resource_manager, - ) - - client.config.add_user_agent("Salt/{}".format(salt.version.__version__)) - - return client - - -def log_cloud_error(client, message, **kwargs): - """ - Log an azurearm cloud error exception - """ - try: - cloud_logger = getattr(log, kwargs.get("azurearm_log_level")) - except (AttributeError, TypeError): - cloud_logger = getattr(log, "error") - - cloud_logger( - "An AzureARM %s CloudError has occurred: %s", client.capitalize(), message - ) - - return - - -def paged_object_to_list(paged_object): - """ - Extract all pages within a paged object as a list of dictionaries - """ - paged_return = [] - while True: - try: - page = next(paged_object) - paged_return.append(page.as_dict()) - except StopIteration: - break - - return paged_return - - -def create_object_model(module_name, object_name, **kwargs): - """ - Assemble an object from incoming parameters. - """ - object_kwargs = {} - - try: - model_module = importlib.import_module( - "azure.mgmt.{}.models".format(module_name) - ) - # pylint: disable=invalid-name - Model = getattr(model_module, object_name) - except ImportError: - raise sys.exit( - "The {} model in the {} Azure module is not available.".format( - object_name, module_name - ) - ) - - if "_attribute_map" in dir(Model): - for attr, items in Model._attribute_map.items(): - param = kwargs.get(attr) - if param is not None: - if items["type"][0].isupper() and isinstance(param, dict): - object_kwargs[attr] = create_object_model( - module_name, items["type"], **param - ) - elif items["type"][0] == "{" and isinstance(param, dict): - object_kwargs[attr] = param - elif items["type"][0] == "[" and isinstance(param, list): - obj_list = [] - for list_item in param: - if items["type"][1].isupper() and isinstance(list_item, dict): - obj_list.append( - create_object_model( - module_name, - items["type"][ - items["type"].index("[") - + 1 : items["type"].rindex("]") - ], - **list_item - ) - ) - elif items["type"][1] == "{" and isinstance(list_item, dict): - obj_list.append(list_item) - elif not items["type"][1].isupper() and items["type"][1] != "{": - obj_list.append(list_item) - object_kwargs[attr] = obj_list - else: - object_kwargs[attr] = param - - # wrap calls to this function to catch TypeError exceptions - return Model(**object_kwargs) - - -def compare_list_of_dicts(old, new, convert_id_to_name=None): - """ - Compare lists of dictionaries representing Azure objects. Only keys found in the "new" dictionaries are compared to - the "old" dictionaries, since getting Azure objects from the API returns some read-only data which should not be - used in the comparison. A list of parameter names can be passed in order to compare a bare object name to a full - Azure ID path for brevity. If string types are found in values, comparison is case insensitive. Return comment - should be used to trigger exit from the calling function. - """ - ret = {} - - if not convert_id_to_name: - convert_id_to_name = [] - - if not isinstance(new, list): - ret["comment"] = "must be provided as a list of dictionaries!" - return ret - - if len(new) != len(old): - ret["changes"] = {"old": old, "new": new} - return ret - - try: - local_configs, remote_configs = ( - sorted(config, key=itemgetter("name")) for config in (new, old) - ) - except TypeError: - ret["comment"] = "configurations must be provided as a list of dictionaries!" - return ret - except KeyError: - ret["comment"] = 'configuration dictionaries must contain the "name" key!' - return ret - - for idx, val in enumerate(local_configs): - for key in val: - local_val = val[key] - if key in convert_id_to_name: - remote_val = ( - remote_configs[idx].get(key, {}).get("id", "").split("/")[-1] - ) - else: - remote_val = remote_configs[idx].get(key) - if isinstance(local_val, str): - local_val = local_val.lower() - if isinstance(remote_val, str): - remote_val = remote_val.lower() - if local_val != remote_val: - ret["changes"] = {"old": remote_configs, "new": local_configs} - return ret - - return ret diff --git a/salt/utils/msazure.py b/salt/utils/msazure.py deleted file mode 100644 index 28f8f33cc2f..00000000000 --- a/salt/utils/msazure.py +++ /dev/null @@ -1,189 +0,0 @@ -""" -.. versionadded:: 2015.8.0 - -Utilities for accessing storage container blobs on Azure -""" - - -import logging - -from salt.exceptions import SaltSystemExit - -HAS_LIBS = False -try: - import azure - - HAS_LIBS = True -except ImportError: - pass - - -log = logging.getLogger(__name__) - - -def get_storage_conn(storage_account=None, storage_key=None, opts=None): - """ - .. versionadded:: 2015.8.0 - - Return a storage_conn object for the storage account - """ - if opts is None: - opts = {} - - if not storage_account: - storage_account = opts.get("storage_account", None) - if not storage_key: - storage_key = opts.get("storage_key", None) - - return azure.storage.BlobService(storage_account, storage_key) - - -def list_blobs(storage_conn=None, **kwargs): - """ - .. versionadded:: 2015.8.0 - - List blobs associated with the container - """ - if not storage_conn: - storage_conn = get_storage_conn(opts=kwargs) - - if "container" not in kwargs: - raise SaltSystemExit( - code=42, msg='An storage container name must be specified as "container"' - ) - - data = storage_conn.list_blobs( - container_name=kwargs["container"], - prefix=kwargs.get("prefix", None), - marker=kwargs.get("marker", None), - maxresults=kwargs.get("maxresults", None), - include=kwargs.get("include", None), - delimiter=kwargs.get("delimiter", None), - ) - - ret = {} - for item in data.blobs: - ret[item.name] = object_to_dict(item) - return ret - - -def put_blob(storage_conn=None, **kwargs): - """ - .. versionadded:: 2015.8.0 - - Upload a blob - """ - if not storage_conn: - storage_conn = get_storage_conn(opts=kwargs) - - if "container" not in kwargs: - raise SaltSystemExit( - code=42, msg='The blob container name must be specified as "container"' - ) - - if "name" not in kwargs: - raise SaltSystemExit(code=42, msg='The blob name must be specified as "name"') - - if "blob_path" not in kwargs and "blob_content" not in kwargs: - raise SaltSystemExit( - code=42, - msg=( - 'Either a path to a file needs to be passed in as "blob_path" ' - 'or the contents of a blob as "blob_content."' - ), - ) - - blob_kwargs = { - "container_name": kwargs["container"], - "blob_name": kwargs["name"], - "cache_control": kwargs.get("cache_control", None), - "content_language": kwargs.get("content_language", None), - "content_md5": kwargs.get("content_md5", None), - "x_ms_blob_content_type": kwargs.get("blob_content_type", None), - "x_ms_blob_content_encoding": kwargs.get("blob_content_encoding", None), - "x_ms_blob_content_language": kwargs.get("blob_content_language", None), - "x_ms_blob_content_md5": kwargs.get("blob_content_md5", None), - "x_ms_blob_cache_control": kwargs.get("blob_cache_control", None), - "x_ms_meta_name_values": kwargs.get("meta_name_values", None), - "x_ms_lease_id": kwargs.get("lease_id", None), - } - if "blob_path" in kwargs: - data = storage_conn.put_block_blob_from_path( - file_path=kwargs["blob_path"], **blob_kwargs - ) - elif "blob_content" in kwargs: - data = storage_conn.put_block_blob_from_bytes( - blob=kwargs["blob_content"], **blob_kwargs - ) - - return data - - -def get_blob(storage_conn=None, **kwargs): - """ - .. versionadded:: 2015.8.0 - - Download a blob - """ - if not storage_conn: - storage_conn = get_storage_conn(opts=kwargs) - - if "container" not in kwargs: - raise SaltSystemExit( - code=42, msg='The blob container name must be specified as "container"' - ) - - if "name" not in kwargs: - raise SaltSystemExit(code=42, msg='The blob name must be specified as "name"') - - if "local_path" not in kwargs and "return_content" not in kwargs: - raise SaltSystemExit( - code=42, - msg=( - 'Either a local path needs to be passed in as "local_path", ' - 'or "return_content" to return the blob contents directly' - ), - ) - - blob_kwargs = { - "container_name": kwargs["container"], - "blob_name": kwargs["name"], - "snapshot": kwargs.get("snapshot", None), - "x_ms_lease_id": kwargs.get("lease_id", None), - "progress_callback": kwargs.get("progress_callback", None), - "max_connections": kwargs.get("max_connections", 1), - "max_retries": kwargs.get("max_retries", 5), - "retry_wait": kwargs.get("retry_wait", 1), - } - - if "local_path" in kwargs: - data = storage_conn.get_blob_to_path( - file_path=kwargs["local_path"], - open_mode=kwargs.get("open_mode", "wb"), - **blob_kwargs - ) - elif "return_content" in kwargs: - data = storage_conn.get_blob_to_bytes(**blob_kwargs) - - return data - - -def object_to_dict(obj): - """ - .. versionadded:: 2015.8.0 - - Convert an object to a dictionary - """ - if isinstance(obj, list) or isinstance(obj, tuple): - ret = [] - for item in obj: - ret.append(object_to_dict(item)) - elif hasattr(obj, "__dict__"): - ret = {} - for item in obj.__dict__: - if item.startswith("_"): - continue - ret[item] = object_to_dict(obj.__dict__[item]) - else: - ret = obj - return ret diff --git a/tests/integration/cloud/clouds/test_msazure.py b/tests/integration/cloud/clouds/test_msazure.py deleted file mode 100644 index 1439d4195d2..00000000000 --- a/tests/integration/cloud/clouds/test_msazure.py +++ /dev/null @@ -1,66 +0,0 @@ -""" - :codeauthor: Nicole Thomas -""" - -import logging - -import pytest - -from salt.utils.versions import Version -from tests.integration.cloud.helpers.cloud_test_base import CloudTest - -try: - import azure # pylint: disable=unused-import - - HAS_AZURE = True -except ImportError: - HAS_AZURE = False - -if HAS_AZURE and not hasattr(azure, "__version__"): - import azure.common - -log = logging.getLogger(__name__) - -TIMEOUT = 1000 -REQUIRED_AZURE = "1.1.0" - - -def __has_required_azure(): - """ - Returns True/False if the required version of the Azure SDK is installed. - """ - if HAS_AZURE: - if hasattr(azure, "__version__"): - version = Version(azure.__version__) - else: - version = Version(azure.common.__version__) - if Version(REQUIRED_AZURE) <= version: - return True - return False - - -@pytest.mark.skipif( - not HAS_AZURE, reason="These tests require the Azure Python SDK to be installed." -) -@pytest.mark.skipif( - not __has_required_azure(), - reason="The Azure Python SDK must be >= {}.".format(REQUIRED_AZURE), -) -class AzureTest(CloudTest): - """ - Integration tests for the Azure cloud provider in Salt-Cloud - """ - - PROVIDER = "azurearm" - REQUIRED_PROVIDER_CONFIG_ITEMS = ("subscription_id",) - - def test_instance(self): - """ - Test creating an instance on Azure - """ - # check if instance with salt installed returned - ret_val = self.run_cloud( - "-p azure-test {}".format(self.instance_name), timeout=TIMEOUT - ) - self.assertInstanceExists(ret_val) - self.assertDestroyInstance(timeout=TIMEOUT) diff --git a/tests/integration/files/conf/cloud.profiles.d/azure.conf b/tests/integration/files/conf/cloud.profiles.d/azure.conf deleted file mode 100644 index bfc749cc9e5..00000000000 --- a/tests/integration/files/conf/cloud.profiles.d/azure.conf +++ /dev/null @@ -1,8 +0,0 @@ -azure-test: - provider: azurearm-config - image: 'b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-14_04-LTS-amd64-server-20140724-en-us-30GB' - size: Standard_D1 - slot: production - ssh_username: '' - ssh_password: '' - script_args: '-P' diff --git a/tests/integration/files/conf/cloud.providers.d/azurearm.conf b/tests/integration/files/conf/cloud.providers.d/azurearm.conf deleted file mode 100644 index 60c34667c1e..00000000000 --- a/tests/integration/files/conf/cloud.providers.d/azurearm.conf +++ /dev/null @@ -1,16 +0,0 @@ -azurearm-config: - driver: azurearm - subscription_id: '' - cleanup_disks: True - cleanup_interfaces: True - cleanup_vhds: True - cleanup_services: True - minion: - master_type: str - username: '' - password: '' - location: '' - network_resource_group: '' - network: '' - subnet: '' - resource_group: '' diff --git a/tests/pytests/unit/cloud/clouds/test_azurearm.py b/tests/pytests/unit/cloud/clouds/test_azurearm.py deleted file mode 100644 index d84aeeffd65..00000000000 --- a/tests/pytests/unit/cloud/clouds/test_azurearm.py +++ /dev/null @@ -1,161 +0,0 @@ -import types - -import pytest - -from salt.cloud.clouds import azurearm as azure -from tests.support.mock import MagicMock, create_autospec, patch - - -def copy_func(func, globals=None): - # I do not know that this is complete, but it's sufficient for now. - # The key to "moving" the function to another module (or stubbed module) - # is to update __globals__. - - copied_func = types.FunctionType( - func.__code__, globals, func.__name__, func.__defaults__, func.__closure__ - ) - copied_func.__module__ = func.__module__ - copied_func.__doc__ = func.__doc__ - copied_func.__kwdefaults__ = func.__kwdefaults__ - copied_func.__dict__.update(func.__dict__) - return copied_func - - -def mock_module(mod, sut=None): - if sut is None: - sut = [None] - - mock = create_autospec(mod) - - # we need to provide a '__globals__' so functions being tested behave correctly. - mock_globals = {} - - # exclude the system under test - for name in sut: - attr = getattr(mod, name) - if isinstance(attr, types.FunctionType): - attr = copy_func(attr, mock_globals) - setattr(mock, name, attr) - - # fully populate our mock_globals - for name in mod.__dict__: - if name in mock.__dict__: - mock_globals[name] = mock.__dict__[name] - elif type(getattr(mod, name)) is type(types): # is a module - mock_globals[name] = getattr(mock, name) - else: - mock_globals[name] = mod.__dict__[name] - - return mock - - -@pytest.fixture -def configure_loader_modules(): - return {azure: {"__opts__": {}, "__active_provider_name__": None}} - - -@pytest.mark.skipif(not azure.HAS_LIBS, reason="azure not available") -def test_function_signatures(): - mock_azure = mock_module(azure, sut=["request_instance", "__opts__", "__utils__"]) - mock_azure.create_network_interface.return_value = [ - MagicMock(), - MagicMock(), - MagicMock(), - ] - mock_azure.salt.utils.stringutils.to_str.return_value = "P4ssw0rd" - mock_azure.salt.utils.cloud.gen_keys.return_value = [MagicMock(), MagicMock()] - mock_azure.__opts__["pki_dir"] = None - - mock_azure.request_instance.__globals__[ - "__builtins__" - ] = mock_azure.request_instance.__globals__["__builtins__"].copy() - mock_azure.request_instance.__globals__["__builtins__"]["getattr"] = MagicMock() - - mock_azure.__utils__["cloud.fire_event"] = mock_azure.salt.utils.cloud.fire_event - mock_azure.__utils__[ - "cloud.filter_event" - ] = mock_azure.salt.utils.cloud.filter_event - mock_azure.__opts__["sock_dir"] = MagicMock() - mock_azure.__opts__["transport"] = MagicMock() - - mock_azure.request_instance( - {"image": "http://img", "storage_account": "blah", "size": ""} - ) - - # we literally only check that a final creation call occurred. - mock_azure.get_conn.return_value.virtual_machines.create_or_update.assert_called_once() - - -def test_get_configured_provider(): - mock_azure = mock_module( - azure, sut=["get_configured_provider", "__opts__", "__utils__"] - ) - - good_combos = [ - { - "subscription_id": "3287abc8-f98a-c678-3bde-326766fd3617", - "tenant": "ABCDEFAB-1234-ABCD-1234-ABCDEFABCDEF", - "client_id": "ABCDEFAB-1234-ABCD-1234-ABCDEFABCDEF", - "secret": "XXXXXXXXXXXXXXXXXXXXXXXX", - }, - { - "subscription_id": "3287abc8-f98a-c678-3bde-326766fd3617", - "username": "larry", - "password": "123pass", - }, - {"subscription_id": "3287abc8-f98a-c678-3bde-326766fd3617"}, - ] - - for combo in good_combos: - mock_azure.__opts__["providers"] = {"azure_test": {"azurearm": combo}} - assert azure.get_configured_provider() == combo - - bad_combos = [ - {"subscrption": "3287abc8-f98a-c678-3bde-326766fd3617"}, - {}, - ] - - for combo in bad_combos: - mock_azure.__opts__["providers"] = {"azure_test": {"azurearm": combo}} - assert not azure.get_configured_provider() - - -def test_get_conn(): - mock_azure = mock_module(azure, sut=["get_conn", "__opts__", "__utils__"]) - - mock_azure.__opts__["providers"] = { - "azure_test": { - "azurearm": { - "subscription_id": "3287abc8-f98a-c678-3bde-326766fd3617", - "driver": "azurearm", - "password": "monkeydonkey", - } - } - } - # password is stripped if username not provided - expected = {"subscription_id": "3287abc8-f98a-c678-3bde-326766fd3617"} - with patch( - "salt.utils.azurearm.get_client", side_effect=lambda client_type, **kw: kw - ): - assert azure.get_conn(client_type="compute") == expected - - mock_azure.__opts__["providers"] = { - "azure_test": { - "azurearm": { - "subscription_id": "3287abc8-f98a-c678-3bde-326766fd3617", - "driver": "azurearm", - "username": "donkeymonkey", - "password": "monkeydonkey", - } - } - } - # username and password via provider config - expected = { - "subscription_id": "3287abc8-f98a-c678-3bde-326766fd3617", - "username": "donkeymonkey", - "password": "monkeydonkey", - } - with patch( - "salt.utils.azurearm.get_client", side_effect=lambda client_type, **kw: kw - ): - assert azure.get_conn(client_type="compute") == expected diff --git a/tests/pytests/unit/grains/test_metadata_azure.py b/tests/pytests/unit/grains/test_metadata_azure.py deleted file mode 100644 index 4f9bc9988a7..00000000000 --- a/tests/pytests/unit/grains/test_metadata_azure.py +++ /dev/null @@ -1,96 +0,0 @@ -""" - Unit test for salt.grains.metadata_azure - - - :codeauthor: :email" `Vishal Gupta - -""" - -import logging - -import pytest - -import salt.grains.metadata_azure as metadata -import salt.utils.http as http -from tests.support.mock import create_autospec, patch - -# from Exception import Exception, ValueError - -log = logging.getLogger(__name__) - - -@pytest.fixture -def configure_loader_modules(): - return {metadata: {"__opts__": {"metadata_server_grains": "True"}}} - - -def test_metadata_azure_search(): - def mock_http(url="", headers=False, header_list=None): - metadata_vals = { - "http://169.254.169.254/metadata/instance?api-version=2020-09-01": { - "body": '{"compute": {"test": "fulltest"}}', - "headers": {"Content-Type": "application/json; charset=utf-8"}, - }, - } - - return metadata_vals[url] - - with patch( - "salt.utils.http.query", - create_autospec(http.query, autospec=True, side_effect=mock_http), - ): - assert metadata.metadata() == {"compute": {"test": "fulltest"}} - - -def test_metadata_virtual(): - print("running 1st") - with patch( - "salt.utils.http.query", - create_autospec( - http.query, - autospec=True, - return_value={ - "error": "Bad request: . Required metadata header not specified" - }, - ), - ): - assert metadata.__virtual__() is False - with patch( - "salt.utils.http.query", - create_autospec( - http.query, - autospec=True, - return_value={ - "body": '{"compute": {"test": "fulltest"}}', - "headers": {"Content-Type": "application/json; charset=utf-8"}, - "status": 200, - }, - ), - ): - assert metadata.__virtual__() is True - with patch( - "salt.utils.http.query", - create_autospec( - http.query, - autospec=True, - return_value={ - "body": "test", - "headers": {"Content-Type": "application/json; charset=utf-8"}, - "status": 404, - }, - ), - ): - assert metadata.__virtual__() is False - with patch( - "salt.utils.http.query", - create_autospec( - http.query, - autospec=True, - return_value={ - "body": "test", - "headers": {"Content-Type": "application/json; charset=utf-8"}, - "status": 400, - }, - ), - ): - assert metadata.__virtual__() is False diff --git a/tests/pytests/unit/modules/test_azurearm_dns.py b/tests/pytests/unit/modules/test_azurearm_dns.py deleted file mode 100644 index 3c09e23143f..00000000000 --- a/tests/pytests/unit/modules/test_azurearm_dns.py +++ /dev/null @@ -1,182 +0,0 @@ -import logging - -import pytest - -import salt.config -import salt.loader -import salt.modules.azurearm_dns as azurearm_dns -from tests.support.mock import MagicMock -from tests.support.sminion import create_sminion - -HAS_LIBS = False -try: - import azure.mgmt.dns.models # pylint: disable=import-error - - HAS_LIBS = True -except ImportError: - HAS_LIBS = False - - -log = logging.getLogger(__name__) - -pytestmark = [ - pytest.mark.skipif( - HAS_LIBS is False, reason="The azure.mgmt.dns module must be installed." - ), -] - - -class AzureObjMock: - """ - mock azure object for as_dict calls - """ - - args = None - kwargs = None - - def __init__(self, args, kwargs, return_value=None): - self.args = args - self.kwargs = kwargs - self.__return_value = return_value - - def __getattr__(self, item): - return self - - def __call__(self, *args, **kwargs): - return MagicMock(return_value=self.__return_value)() - - def as_dict(self, *args, **kwargs): - return self.args, self.kwargs - - -class AzureFuncMock: - """ - mock azure client function calls - """ - - def __init__(self, return_value=None): - self.__return_value = return_value - - def __getattr__(self, item): - return self - - def __call__(self, *args, **kwargs): - return MagicMock(return_value=self.__return_value)() - - def create_or_update(self, *args, **kwargs): - azure_obj = AzureObjMock(args, kwargs) - return azure_obj - - -class AzureSubMock: - """ - mock azure client sub-modules - """ - - record_sets = AzureFuncMock() - zones = AzureFuncMock() - - def __init__(self, return_value=None): - self.__return_value = return_value - - def __getattr__(self, item): - return self - - def __call__(self, *args, **kwargs): - return MagicMock(return_value=self.__return_value)() - - -class AzureClientMock: - """ - mock azure client - """ - - def __init__(self, return_value=AzureSubMock): - self.__return_value = return_value - - def __getattr__(self, item): - return self - - def __call__(self, *args, **kwargs): - return MagicMock(return_value=self.__return_value)() - - -@pytest.fixture -def credentials(): - azurearm_dns.__virtual__() - return { - "client_id": "CLIENT_ID", - "secret": "SECRET", - "subscription_id": "SUBSCRIPTION_ID", - "tenant": "TENANT", - } - - -@pytest.fixture -def configure_loader_modules(): - """ - setup loader modules and override the azurearm.get_client utility - """ - minion_config = create_sminion().opts.copy() - utils = salt.loader.utils(minion_config) - funcs = salt.loader.minion_mods( - minion_config, utils=utils, whitelist=["azurearm_dns", "config"] - ) - utils["azurearm.get_client"] = AzureClientMock() - return { - azurearm_dns: {"__utils__": utils, "__salt__": funcs}, - } - - -def test_record_set_create_or_update(credentials): - """ - tests record set object creation - """ - expected = { - "if_match": None, - "if_none_match": None, - "parameters": {"arecords": [{"ipv4_address": "10.0.0.1"}], "ttl": 300}, - "record_type": "A", - "relative_record_set_name": "myhost", - "resource_group_name": "testgroup", - "zone_name": "myzone", - } - - record_set_args, record_set_kwargs = azurearm_dns.record_set_create_or_update( - "myhost", - "myzone", - "testgroup", - "A", - arecords=[{"ipv4_address": "10.0.0.1"}], - ttl=300, - **credentials - ) - - for key, val in record_set_kwargs.items(): - if isinstance(val, azure.mgmt.dns.models.RecordSet): - record_set_kwargs[key] = val.as_dict() - - assert record_set_kwargs == expected - - -def test_zone_create_or_update(credentials): - """ - tests zone object creation - """ - expected = { - "if_match": None, - "if_none_match": None, - "parameters": {"location": "global", "zone_type": "Public"}, - "resource_group_name": "testgroup", - "zone_name": "myzone", - } - - zone_args, zone_kwargs = azurearm_dns.zone_create_or_update( - "myzone", "testgroup", **credentials - ) - - for key, val in zone_kwargs.items(): - if isinstance(val, azure.mgmt.dns.models.Zone): - zone_kwargs[key] = val.as_dict() - - assert zone_kwargs == expected diff --git a/tests/pytests/unit/pillar/test_azureblob.py b/tests/pytests/unit/pillar/test_azureblob.py deleted file mode 100644 index b152f614bdc..00000000000 --- a/tests/pytests/unit/pillar/test_azureblob.py +++ /dev/null @@ -1,333 +0,0 @@ -""" -Tests for the Azure Blob External Pillar. -""" - -import pickle -import time - -import pytest - -import salt.config -import salt.loader -import salt.pillar.azureblob as azureblob -import salt.utils.files -from tests.support.mock import MagicMock, patch - -HAS_LIBS = False -try: - # pylint: disable=no-name-in-module - from azure.storage.blob import BlobServiceClient - - # pylint: enable=no-name-in-module - - HAS_LIBS = True -except ImportError: - pass - - -pytestmark = [ - pytest.mark.skipif( - HAS_LIBS is False, - reason="The azure.storage.blob module must be installed.", - ) -] - - -class MockBlob(dict): - """ - Creates a Mock Blob object. - """ - - name = "" - - def __init__(self): - super().__init__( - { - "container": None, - "name": "test.sls", - "prefix": None, - "delimiter": "/", - "results_per_page": None, - "location_mode": None, - } - ) - - -class MockContainerClient: - """ - Creates a Mock ContainerClient. - """ - - def __init__(self): - pass - - def walk_blobs(self, *args, **kwargs): - yield MockBlob() - - def get_blob_client(self, *args, **kwargs): - pass - - -class MockBlobServiceClient: - """ - Creates a Mock BlobServiceClient. - """ - - def __init__(self): - pass - - def get_container_client(self, *args, **kwargs): - container_client = MockContainerClient() - return container_client - - -@pytest.fixture -def cachedir(tmp_path): - dirname = tmp_path / "cachedir" - dirname.mkdir(parents=True, exist_ok=True) - return dirname - - -@pytest.fixture -def configure_loader_modules(cachedir, tmp_path): - base_pillar = tmp_path / "base" - prod_pillar = tmp_path / "prod" - base_pillar.mkdir(parents=True, exist_ok=True) - prod_pillar.mkdir(parents=True, exist_ok=True) - pillar_roots = { - "base": [str(base_pillar)], - "prod": [str(prod_pillar)], - } - opts = { - "cachedir": cachedir, - "pillar_roots": pillar_roots, - } - return { - azureblob: {"__opts__": opts}, - } - - -def test__init_expired(tmp_path): - """ - Tests the result of _init when the cache is expired. - """ - container = "test" - multiple_env = False - environment = "base" - blob_cache_expire = 0 # The cache will be expired - blob_client = MockBlobServiceClient() - cache_file = tmp_path / "cache_file" - # Patches the _get_containers_cache_filename module so that it returns the name of the new tempfile that - # represents the cache file - with patch.object( - azureblob, - "_get_containers_cache_filename", - MagicMock(return_value=str(cache_file)), - ): - # Patches the from_connection_string module of the BlobServiceClient class so that a connection string does - # not need to be given. Additionally it returns example blob data used by the ext_pillar. - with patch.object( - BlobServiceClient, - "from_connection_string", - MagicMock(return_value=blob_client), - ): - ret = azureblob._init( - "", container, multiple_env, environment, blob_cache_expire - ) - - expected = { - "base": { - "test": [ - { - "container": None, - "name": "test.sls", - "prefix": None, - "delimiter": "/", - "results_per_page": None, - "location_mode": None, - } - ] - } - } - assert ret == expected - - -def test__init_not_expired(tmp_path): - """ - Tests the result of _init when the cache is not expired. - """ - container = "test" - multiple_env = False - environment = "base" - blob_cache_expire = (time.time()) * (time.time()) # The cache will not be expired - metadata = { - "base": { - "test": [ - {"name": "base/secret.sls", "relevant": "include.sls"}, - {"name": "blobtest.sls", "irrelevant": "ignore.sls"}, - ] - } - } - cache_file = tmp_path / "cache_file" - # Pickles the metadata and stores it in cache_file - with salt.utils.files.fopen(str(cache_file), "wb") as fp_: - pickle.dump(metadata, fp_) - # Patches the _get_containers_cache_filename module so that it returns the name of the new tempfile that - # represents the cache file - with patch.object( - azureblob, - "_get_containers_cache_filename", - MagicMock(return_value=str(cache_file)), - ): - # Patches the _read_containers_cache_file module so that it returns what it normally would if the new - # tempfile representing the cache file was passed to it - plugged = azureblob._read_containers_cache_file(str(cache_file)) - with patch.object( - azureblob, - "_read_containers_cache_file", - MagicMock(return_value=plugged), - ): - ret = azureblob._init( - "", container, multiple_env, environment, blob_cache_expire - ) - assert ret == metadata - - -def test__get_cache_dir(cachedir): - """ - Tests the result of _get_cache_dir. - """ - ret = azureblob._get_cache_dir() - assert ret == str(cachedir / "pillar_azureblob") - - -def test__get_cached_file_name(cachedir): - """ - Tests the result of _get_cached_file_name. - """ - container = "test" - saltenv = "base" - path = "base/secret.sls" - ret = azureblob._get_cached_file_name(container, saltenv, path) - assert ret == str(cachedir / "pillar_azureblob" / saltenv / container / path) - - -def test__get_containers_cache_filename(cachedir): - """ - Tests the result of _get_containers_cache_filename. - """ - container = "test" - ret = azureblob._get_containers_cache_filename(container) - assert ret == str(cachedir / "pillar_azureblob" / "test-files.cache") - - -def test__refresh_containers_cache_file(tmp_path): - """ - Tests the result of _refresh_containers_cache_file to ensure that it successfully copies blob data into a - cache file. - """ - blob_client = MockBlobServiceClient() - container = "test" - cache_file = tmp_path / "cache_file" - with patch.object( - BlobServiceClient, - "from_connection_string", - MagicMock(return_value=blob_client), - ): - ret = azureblob._refresh_containers_cache_file("", container, str(cache_file)) - expected = { - "base": { - "test": [ - { - "container": None, - "name": "test.sls", - "prefix": None, - "delimiter": "/", - "results_per_page": None, - "location_mode": None, - } - ] - } - } - assert ret == expected - - -def test__read_containers_cache_file(tmp_path): - """ - Tests the result of _read_containers_cache_file to make sure that it successfully loads in pickled metadata. - """ - metadata = { - "base": { - "test": [ - {"name": "base/secret.sls", "relevant": "include.sls"}, - {"name": "blobtest.sls", "irrelevant": "ignore.sls"}, - ] - } - } - cache_file = tmp_path / "cache_file" - # Pickles the metadata and stores it in cache_file - with salt.utils.files.fopen(str(cache_file), "wb") as fp_: - pickle.dump(metadata, fp_) - # Checks to see if _read_containers_cache_file can successfully read the pickled metadata from the cache file - ret = azureblob._read_containers_cache_file(str(cache_file)) - assert ret == metadata - - -def test__find_files(): - """ - Tests the result of _find_files. Ensures it only finds files and not directories. Ensures it also ignore - irrelevant files. - """ - metadata = { - "test": [ - {"name": "base/secret.sls"}, - {"name": "blobtest.sls", "irrelevant": "ignore.sls"}, - {"name": "base/"}, - ] - } - ret = azureblob._find_files(metadata) - assert ret == {"test": ["base/secret.sls", "blobtest.sls"]} - - -def test__find_file_meta1(): - """ - Tests the result of _find_file_meta when the metadata contains a blob with the specified path and a blob - without the specified path. - """ - metadata = { - "base": { - "test": [ - {"name": "base/secret.sls", "relevant": "include.sls"}, - {"name": "blobtest.sls", "irrelevant": "ignore.sls"}, - ] - } - } - container = "test" - saltenv = "base" - path = "base/secret.sls" - ret = azureblob._find_file_meta(metadata, container, saltenv, path) - assert ret == {"name": "base/secret.sls", "relevant": "include.sls"} - - -def test__find_file_meta2(): - """ - Tests the result of _find_file_meta when the saltenv in metadata does not match the specified saltenv. - """ - metadata = {"wrong": {"test": [{"name": "base/secret.sls"}]}} - container = "test" - saltenv = "base" - path = "base/secret.sls" - ret = azureblob._find_file_meta(metadata, container, saltenv, path) - assert ret is None - - -def test__find_file_meta3(): - """ - Tests the result of _find_file_meta when the container in metadata does not match the specified metadata. - """ - metadata = {"base": {"wrong": [{"name": "base/secret.sls"}]}} - container = "test" - saltenv = "base" - path = "base/secret.sls" - ret = azureblob._find_file_meta(metadata, container, saltenv, path) - assert ret is None diff --git a/tests/unit/utils/test_azurearm.py b/tests/unit/utils/test_azurearm.py deleted file mode 100644 index 6bfab788fae..00000000000 --- a/tests/unit/utils/test_azurearm.py +++ /dev/null @@ -1,55 +0,0 @@ -import logging - -import pytest - -import salt.utils.azurearm as azurearm -from tests.support.unit import TestCase - -# Azure libs -# pylint: disable=import-error -HAS_LIBS = False -try: - import azure.mgmt.compute.models # pylint: disable=unused-import - import azure.mgmt.network.models # pylint: disable=unused-import - - HAS_LIBS = True -except ImportError: - pass - -# pylint: enable=import-error - -log = logging.getLogger(__name__) - -MOCK_CREDENTIALS = { - "client_id": "CLIENT_ID", - "secret": "SECRET", - "subscription_id": "SUBSCRIPTION_ID", - "tenant": "TENANT", -} - - -@pytest.mark.skipif( - HAS_LIBS is False, reason="The azure.mgmt.network module must be installed." -) -class AzureRmUtilsTestCase(TestCase): - def test_create_object_model_vnet(self): - module_name = "network" - object_name = "VirtualNetwork" - vnet = { - "address_space": {"address_prefixes": ["10.0.0.0/8"]}, - "enable_ddos_protection": False, - "enable_vm_protection": True, - "tags": {"contact_name": "Elmer Fudd Gantry"}, - } - model = azurearm.create_object_model(module_name, object_name, **vnet) - self.assertEqual(vnet, model.as_dict()) - - def test_create_object_model_nic_ref(self): - module_name = "compute" - object_name = "NetworkInterfaceReference" - ref = { - "id": "/subscriptions/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic", - "primary": False, - } - model = azurearm.create_object_model(module_name, object_name, **ref) - self.assertEqual(ref, model.as_dict()) From df28d77fbcb5876c6157f0ee7bca508658f498a3 Mon Sep 17 00:00:00 2001 From: natalieswork Date: Mon, 22 May 2023 14:23:44 -0400 Subject: [PATCH 022/152] adding .env to git ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f4076ae84be..405f58704b1 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ Pipfile.lock # - /some/path$ git clone https://github.com/thatch45/salt.git # - /some/path$ virtualenv --python=/usr/bin/python2.6 salt /env/ +/.env/ /bin/ /etc/ /include/ From 82ba2a5da83206710b0f3de1dee771085f34a075 Mon Sep 17 00:00:00 2001 From: Tanmoy037 Date: Sun, 21 May 2023 11:45:59 +0530 Subject: [PATCH 023/152] remove the header comment about masterless --- doc/ref/configuration/minion.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 69c7afbde84..256951d2849 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -2035,7 +2035,6 @@ Valid options: Top File Settings ================= -These parameters only have an effect if running a masterless minion. .. conf_minion:: state_top From b713c3441b45914a7ca8eefe64df5074eddf67c9 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 5 May 2023 22:08:19 +0100 Subject: [PATCH 024/152] Pass the `LATEST_SALT_RELEASE` environment variables through to the VM Signed-off-by: Pedro Algarvio --- .github/workflows/test-package-downloads-action-linux.yml | 4 ++-- .github/workflows/test-package-downloads-action-windows.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-package-downloads-action-linux.yml b/.github/workflows/test-package-downloads-action-linux.yml index ee67c4d4020..7df9ec1c8f3 100644 --- a/.github/workflows/test-package-downloads-action-linux.yml +++ b/.github/workflows/test-package-downloads-action-linux.yml @@ -224,7 +224,7 @@ jobs: run: | tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \ -E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING \ + -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE \ --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }} -- download-pkgs - name: Run Package Download Tests @@ -241,7 +241,7 @@ jobs: run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ -E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING \ + -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE \ --nox-session=${{ inputs.nox-session }} --rerun-failures ${{ inputs.distro-slug }} -- download-pkgs - name: Combine Coverage Reports diff --git a/.github/workflows/test-package-downloads-action-windows.yml b/.github/workflows/test-package-downloads-action-windows.yml index 10d4462e451..963372925d2 100644 --- a/.github/workflows/test-package-downloads-action-windows.yml +++ b/.github/workflows/test-package-downloads-action-windows.yml @@ -234,7 +234,7 @@ jobs: run: | tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \ -E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING \ + -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE \ --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }} -- download-pkgs - name: Run Package Download Tests @@ -252,7 +252,7 @@ jobs: run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ -E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING \ + -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE \ --nox-session=${{ inputs.nox-session }} --rerun-failures ${{ inputs.distro-slug }} -- download-pkgs - name: Combine Coverage Reports From 75a93eefc3d2b606ff00cc8764b1cbbbffe710c2 Mon Sep 17 00:00:00 2001 From: MKLeb Date: Tue, 2 May 2023 20:53:24 -0400 Subject: [PATCH 025/152] Refactor the `tools pkg repo` commands into a subdirectory --- tools/__init__.py | 2 + tools/pkg/repo.py | 1906 ------------------------------------ tools/pkg/repo/__init__.py | 181 ++++ tools/pkg/repo/create.py | 1038 ++++++++++++++++++++ tools/pkg/repo/publish.py | 653 ++++++++++++ tools/utils.py | 127 +++ 6 files changed, 2001 insertions(+), 1906 deletions(-) delete mode 100644 tools/pkg/repo.py create mode 100644 tools/pkg/repo/__init__.py create mode 100644 tools/pkg/repo/create.py create mode 100644 tools/pkg/repo/publish.py diff --git a/tools/__init__.py b/tools/__init__.py index 419ec309c2f..02e6b8de903 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -8,6 +8,8 @@ ptscripts.register_tools_module("tools.docs") ptscripts.register_tools_module("tools.pkg") ptscripts.register_tools_module("tools.pkg.repo") ptscripts.register_tools_module("tools.pkg.build") +ptscripts.register_tools_module("tools.pkg.repo.create") +ptscripts.register_tools_module("tools.pkg.repo.publish") ptscripts.register_tools_module("tools.pre_commit") ptscripts.register_tools_module("tools.release") ptscripts.register_tools_module("tools.vm") diff --git a/tools/pkg/repo.py b/tools/pkg/repo.py deleted file mode 100644 index d781cf3c8ff..00000000000 --- a/tools/pkg/repo.py +++ /dev/null @@ -1,1906 +0,0 @@ -""" -These commands are used to build the pacakge repository files. -""" -# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated -from __future__ import annotations - -import fnmatch -import hashlib -import json -import logging -import os -import pathlib -import re -import shutil -import sys -import tempfile -import textwrap -from datetime import datetime -from typing import TYPE_CHECKING, Any - -import packaging.version -from ptscripts import Context, command_group - -import tools.pkg -import tools.utils -from tools.utils import Version, get_salt_releases - -try: - import boto3 - from botocore.exceptions import ClientError -except ImportError: - print( - "\nPlease run 'python -m pip install -r " - "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info), - file=sys.stderr, - flush=True, - ) - raise - -log = logging.getLogger(__name__) - -# Define the command group -repo = command_group( - name="repo", - help="Packaging Repository Related Commands", - description=__doc__, - parent="pkg", -) - -create = command_group( - name="create", help="Packaging Repository Creation Related Commands", parent=repo -) - -publish = command_group( - name="publish", - help="Packaging Repository Publication Related Commands", - parent=repo, -) - - -_deb_distro_info = { - "debian": { - "10": { - "label": "deb10ary", - "codename": "buster", - "suitename": "oldstable", - }, - "11": { - "label": "deb11ary", - "codename": "bullseye", - "suitename": "stable", - }, - }, - "ubuntu": { - "20.04": { - "label": "salt_ubuntu2004", - "codename": "focal", - }, - "22.04": { - "label": "salt_ubuntu2204", - "codename": "jammy", - }, - }, -} - - -@create.command( - name="deb", - arguments={ - "salt_version": { - "help": ( - "The salt version for which to build the repository configuration files. " - "If not passed, it will be discovered by running 'python3 salt/version.py'." - ), - "required": True, - }, - "distro": { - "help": "The debian based distribution to build the repository for", - "choices": list(_deb_distro_info), - "required": True, - }, - "distro_version": { - "help": "The distro version.", - "required": True, - }, - "distro_arch": { - "help": "The distribution architecture", - "choices": ("x86_64", "amd64", "aarch64", "arm64"), - }, - "repo_path": { - "help": "Path where the repository shall be created.", - "required": True, - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "incoming": { - "help": ( - "The path to the directory containing the files that should added to " - "the repository." - ), - "required": True, - }, - "nightly_build_from": { - "help": "Developement repository target", - }, - }, -) -def debian( - ctx: Context, - salt_version: str = None, - distro: str = None, - distro_version: str = None, - incoming: pathlib.Path = None, - repo_path: pathlib.Path = None, - key_id: str = None, - distro_arch: str = "amd64", - nightly_build_from: str = None, -): - """ - Create the debian repository. - """ - if TYPE_CHECKING: - assert salt_version is not None - assert distro is not None - assert distro_version is not None - assert incoming is not None - assert repo_path is not None - assert key_id is not None - display_name = f"{distro.capitalize()} {distro_version}" - if distro_version not in _deb_distro_info[distro]: - ctx.error(f"Support for {display_name} is missing.") - ctx.exit(1) - - if distro_arch == "x86_64": - ctx.info(f"The {distro_arch} arch is an alias for 'amd64'. Adjusting.") - distro_arch = "amd64" - - if distro_arch == "aarch64": - ctx.info(f"The {distro_arch} arch is an alias for 'arm64'. Adjusting.") - distro_arch = "arm64" - - distro_details = _deb_distro_info[distro][distro_version] - - ctx.info("Distribution Details:") - ctx.info(distro_details) - if TYPE_CHECKING: - assert isinstance(distro_details["label"], str) - assert isinstance(distro_details["codename"], str) - assert isinstance(distro_details["suitename"], str) - label: str = distro_details["label"] - codename: str = distro_details["codename"] - - ftp_archive_config_suite = "" - if distro == "debian": - suitename: str = distro_details["suitename"] - ftp_archive_config_suite = ( - f"""\n APT::FTPArchive::Release::Suite "{suitename}";\n""" - ) - archive_description = f"SaltProject {display_name} Python 3{'' if not nightly_build_from else ' development'} Salt package repo" - ftp_archive_config = f"""\ - APT::FTPArchive::Release::Origin "SaltProject"; - APT::FTPArchive::Release::Label "{label}";{ftp_archive_config_suite} - APT::FTPArchive::Release::Codename "{codename}"; - APT::FTPArchive::Release::Architectures "{distro_arch}"; - APT::FTPArchive::Release::Components "main"; - APT::FTPArchive::Release::Description "{archive_description}"; - APT::FTPArchive::Release::Acquire-By-Hash "yes"; - Dir {{ - ArchiveDir "."; - }}; - BinDirectory "pool" {{ - Packages "dists/{codename}/main/binary-{distro_arch}/Packages"; - Sources "dists/{codename}/main/source/Sources"; - Contents "dists/{codename}/main/Contents-{distro_arch}"; - }} - """ - ctx.info("Creating repository directory structure ...") - create_repo_path = _create_top_level_repo_path( - ctx, - repo_path, - salt_version, - distro, - distro_version=distro_version, - distro_arch=distro_arch, - nightly_build_from=nightly_build_from, - ) - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - - create_repo_path = _create_repo_path( - ctx, - repo_path, - salt_version, - distro, - distro_version=distro_version, - distro_arch=distro_arch, - nightly_build_from=nightly_build_from, - ) - ftp_archive_config_file = create_repo_path / "apt-ftparchive.conf" - ctx.info(f"Writing {ftp_archive_config_file} ...") - ftp_archive_config_file.write_text(textwrap.dedent(ftp_archive_config)) - - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - - pool_path = create_repo_path / "pool" - pool_path.mkdir(exist_ok=True) - for fpath in incoming.iterdir(): - dpath = pool_path / fpath.name - ctx.info(f"Copying {fpath} to {dpath} ...") - shutil.copyfile(fpath, dpath) - if fpath.suffix == ".dsc": - ctx.info(f"Running 'debsign' on {dpath} ...") - ctx.run("debsign", "--re-sign", "-k", key_id, str(dpath), interactive=True) - - dists_path = create_repo_path / "dists" - symlink_parent_path = dists_path / codename / "main" - symlink_paths = ( - symlink_parent_path / "by-hash" / "SHA256", - symlink_parent_path / "source" / "by-hash" / "SHA256", - symlink_parent_path / f"binary-{distro_arch}" / "by-hash" / "SHA256", - ) - - for path in symlink_paths: - path.mkdir(exist_ok=True, parents=True) - - cmdline = ["apt-ftparchive", "generate", "apt-ftparchive.conf"] - ctx.info(f"Running '{' '.join(cmdline)}' ...") - ctx.run(*cmdline, cwd=create_repo_path) - - ctx.info("Creating by-hash symlinks ...") - for path in symlink_paths: - for fpath in path.parent.parent.iterdir(): - if not fpath.is_file(): - continue - sha256sum = ctx.run("sha256sum", str(fpath), capture=True) - link = path / sha256sum.stdout.decode().split()[0] - link.symlink_to(f"../../{fpath.name}") - - cmdline = [ - "apt-ftparchive", - "--no-md5", - "--no-sha1", - "--no-sha512", - "release", - "-c", - "apt-ftparchive.conf", - f"dists/{codename}/", - ] - ctx.info(f"Running '{' '.join(cmdline)}' ...") - ret = ctx.run(*cmdline, capture=True, cwd=create_repo_path) - release_file = dists_path / codename / "Release" - ctx.info(f"Writing {release_file} with the output of the previous command...") - release_file.write_bytes(ret.stdout) - - cmdline = [ - "gpg", - "-u", - key_id, - "-o", - f"dists/{codename}/InRelease", - "-a", - "-s", - "--clearsign", - f"dists/{codename}/Release", - ] - ctx.info(f"Running '{' '.join(cmdline)}' ...") - ctx.run(*cmdline, cwd=create_repo_path) - - cmdline = [ - "gpg", - "-u", - key_id, - "-o", - f"dists/{codename}/Release.gpg", - "-a", - "-b", - "-s", - f"dists/{codename}/Release", - ] - - ctx.info(f"Running '{' '.join(cmdline)}' ...") - ctx.run(*cmdline, cwd=create_repo_path) - if not nightly_build_from: - remote_versions = _get_remote_versions( - tools.utils.STAGING_BUCKET_NAME, - create_repo_path.parent.relative_to(repo_path), - ) - major_version = Version(salt_version).major - matching_major = None - for version in remote_versions: - if version.major == major_version: - matching_major = version - break - if not matching_major or matching_major <= salt_version: - major_link = create_repo_path.parent.parent / str(major_version) - ctx.info(f"Creating '{major_link.relative_to(repo_path)}' symlink ...") - major_link.symlink_to(f"minor/{salt_version}") - if not remote_versions or remote_versions[0] <= salt_version: - latest_link = create_repo_path.parent.parent / "latest" - ctx.info(f"Creating '{latest_link.relative_to(repo_path)}' symlink ...") - latest_link.symlink_to(f"minor/{salt_version}") - - ctx.info("Done") - - -_rpm_distro_info = { - "amazon": ["2"], - "redhat": ["7", "8", "9"], - "fedora": ["36", "37", "38"], - "photon": ["3", "4"], -} - - -@create.command( - name="rpm", - arguments={ - "salt_version": { - "help": ( - "The salt version for which to build the repository configuration files. " - "If not passed, it will be discovered by running 'python3 salt/version.py'." - ), - "required": True, - }, - "distro": { - "help": "The debian based distribution to build the repository for", - "choices": list(_rpm_distro_info), - "required": True, - }, - "distro_version": { - "help": "The distro version.", - "required": True, - }, - "distro_arch": { - "help": "The distribution architecture", - "choices": ("x86_64", "aarch64", "arm64"), - }, - "repo_path": { - "help": "Path where the repository shall be created.", - "required": True, - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "incoming": { - "help": ( - "The path to the directory containing the files that should added to " - "the repository." - ), - "required": True, - }, - "nightly_build_from": { - "help": "Developement repository target", - }, - }, -) -def rpm( - ctx: Context, - salt_version: str = None, - distro: str = None, - distro_version: str = None, - incoming: pathlib.Path = None, - repo_path: pathlib.Path = None, - key_id: str = None, - distro_arch: str = "amd64", - nightly_build_from: str = None, -): - """ - Create the redhat repository. - """ - if TYPE_CHECKING: - assert salt_version is not None - assert distro is not None - assert distro_version is not None - assert incoming is not None - assert repo_path is not None - assert key_id is not None - display_name = f"{distro.capitalize()} {distro_version}" - if distro_version not in _rpm_distro_info[distro]: - ctx.error(f"Support for {display_name} is missing.") - ctx.exit(1) - - if distro_arch == "aarch64": - ctx.info(f"The {distro_arch} arch is an alias for 'arm64'. Adjusting.") - distro_arch = "arm64" - - ctx.info("Creating repository directory structure ...") - create_repo_path = _create_top_level_repo_path( - ctx, - repo_path, - salt_version, - distro, - distro_version=distro_version, - distro_arch=distro_arch, - nightly_build_from=nightly_build_from, - ) - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - - create_repo_path = _create_repo_path( - ctx, - repo_path, - salt_version, - distro, - distro_version=distro_version, - distro_arch=distro_arch, - nightly_build_from=nightly_build_from, - ) - - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - - for fpath in incoming.iterdir(): - if ".src" in fpath.suffixes: - dpath = create_repo_path / "SRPMS" / fpath.name - else: - dpath = create_repo_path / fpath.name - ctx.info(f"Copying {fpath} to {dpath} ...") - shutil.copyfile(fpath, dpath) - if fpath.suffix == ".rpm": - ctx.info(f"Running 'rpmsign' on {dpath} ...") - ctx.run( - "rpmsign", - "--key-id", - key_id, - "--addsign", - "--digest-algo=sha256", - str(dpath), - ) - - createrepo = shutil.which("createrepo") - if createrepo is None: - container = "ghcr.io/saltstack/salt-ci-containers/packaging:centosstream-9" - ctx.info(f"Using docker container '{container}' to call 'createrepo'...") - uid = ctx.run("id", "-u", capture=True).stdout.strip().decode() - gid = ctx.run("id", "-g", capture=True).stdout.strip().decode() - ctx.run( - "docker", - "run", - "--rm", - "-v", - f"{create_repo_path.resolve()}:/code", - "-u", - f"{uid}:{gid}", - "-w", - "/code", - container, - "createrepo", - ".", - ) - else: - ctx.run("createrepo", ".", cwd=create_repo_path) - - if nightly_build_from: - repo_domain = os.environ.get("SALT_REPO_DOMAIN_RELEASE", "repo.saltproject.io") - else: - repo_domain = os.environ.get( - "SALT_REPO_DOMAIN_STAGING", "staging.repo.saltproject.io" - ) - - salt_repo_user = os.environ.get("SALT_REPO_USER") - if salt_repo_user: - log.info( - "SALT_REPO_USER: %s", - salt_repo_user[0] + "*" * (len(salt_repo_user) - 2) + salt_repo_user[-1], - ) - salt_repo_pass = os.environ.get("SALT_REPO_PASS") - if salt_repo_pass: - log.info( - "SALT_REPO_PASS: %s", - salt_repo_pass[0] + "*" * (len(salt_repo_pass) - 2) + salt_repo_pass[-1], - ) - if salt_repo_user and salt_repo_pass: - repo_domain = f"{salt_repo_user}:{salt_repo_pass}@{repo_domain}" - - def _create_repo_file(create_repo_path, url_suffix): - ctx.info(f"Creating '{repo_file_path.relative_to(repo_path)}' file ...") - if nightly_build_from: - base_url = f"salt-dev/{nightly_build_from}/" - repo_file_contents = "[salt-nightly-repo]" - elif "rc" in salt_version: - base_url = "salt_rc/" - repo_file_contents = "[salt-rc-repo]" - else: - base_url = "" - repo_file_contents = "[salt-repo]" - base_url += f"salt/py3/{distro}/{distro_version}/{distro_arch}/{url_suffix}" - if distro == "amazon": - distro_name = "Amazon Linux" - elif distro == "redhat": - distro_name = "RHEL/CentOS" - else: - distro_name = distro.capitalize() - - if distro != "photon" and int(distro_version) < 8: - failovermethod = "\n failovermethod=priority" - else: - failovermethod = "" - - repo_file_contents += textwrap.dedent( - f""" - name=Salt repo for {distro_name} {distro_version} PY3 - baseurl=https://{repo_domain}/{base_url} - skip_if_unavailable=True{failovermethod} - priority=10 - enabled=1 - enabled_metadata=1 - gpgcheck=1 - gpgkey=https://{repo_domain}/{base_url}/{tools.utils.GPG_KEY_FILENAME}.pub - """ - ) - create_repo_path.write_text(repo_file_contents) - - if nightly_build_from: - repo_file_path = create_repo_path.parent / "nightly.repo" - else: - repo_file_path = create_repo_path.parent / f"{create_repo_path.name}.repo" - - _create_repo_file(repo_file_path, f"minor/{salt_version}") - - if not nightly_build_from: - remote_versions = _get_remote_versions( - tools.utils.STAGING_BUCKET_NAME, - create_repo_path.parent.relative_to(repo_path), - ) - major_version = Version(salt_version).major - matching_major = None - for version in remote_versions: - if version.major == major_version: - matching_major = version - break - if not matching_major or matching_major <= salt_version: - major_link = create_repo_path.parent.parent / str(major_version) - ctx.info(f"Creating '{major_link.relative_to(repo_path)}' symlink ...") - major_link.symlink_to(f"minor/{salt_version}") - repo_file_path = create_repo_path.parent.parent / f"{major_version}.repo" - _create_repo_file(repo_file_path, str(major_version)) - if not remote_versions or remote_versions[0] <= salt_version: - latest_link = create_repo_path.parent.parent / "latest" - ctx.info(f"Creating '{latest_link.relative_to(repo_path)}' symlink ...") - latest_link.symlink_to(f"minor/{salt_version}") - repo_file_path = create_repo_path.parent.parent / "latest.repo" - _create_repo_file(repo_file_path, "latest") - - ctx.info("Done") - - -@create.command( - name="windows", - arguments={ - "salt_version": { - "help": "The salt version for which to build the repository", - "required": True, - }, - "repo_path": { - "help": "Path where the repository shall be created.", - "required": True, - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "incoming": { - "help": ( - "The path to the directory containing the files that should added to " - "the repository." - ), - "required": True, - }, - "nightly_build_from": { - "help": "Developement repository target", - }, - }, -) -def windows( - ctx: Context, - salt_version: str = None, - incoming: pathlib.Path = None, - repo_path: pathlib.Path = None, - key_id: str = None, - nightly_build_from: str = None, -): - """ - Create the windows repository. - """ - if TYPE_CHECKING: - assert salt_version is not None - assert incoming is not None - assert repo_path is not None - assert key_id is not None - _create_onedir_based_repo( - ctx, - salt_version=salt_version, - nightly_build_from=nightly_build_from, - repo_path=repo_path, - incoming=incoming, - key_id=key_id, - distro="windows", - pkg_suffixes=(".msi", ".exe"), - ) - ctx.info("Done") - - -@create.command( - name="macos", - arguments={ - "salt_version": { - "help": "The salt version for which to build the repository", - "required": True, - }, - "repo_path": { - "help": "Path where the repository shall be created.", - "required": True, - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "incoming": { - "help": ( - "The path to the directory containing the files that should added to " - "the repository." - ), - "required": True, - }, - "nightly_build_from": { - "help": "Developement repository target", - }, - }, -) -def macos( - ctx: Context, - salt_version: str = None, - incoming: pathlib.Path = None, - repo_path: pathlib.Path = None, - key_id: str = None, - nightly_build_from: str = None, -): - """ - Create the windows repository. - """ - if TYPE_CHECKING: - assert salt_version is not None - assert incoming is not None - assert repo_path is not None - assert key_id is not None - _create_onedir_based_repo( - ctx, - salt_version=salt_version, - nightly_build_from=nightly_build_from, - repo_path=repo_path, - incoming=incoming, - key_id=key_id, - distro="macos", - pkg_suffixes=(".pkg",), - ) - ctx.info("Done") - - -@create.command( - name="onedir", - arguments={ - "salt_version": { - "help": "The salt version for which to build the repository", - "required": True, - }, - "repo_path": { - "help": "Path where the repository shall be created.", - "required": True, - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "incoming": { - "help": ( - "The path to the directory containing the files that should added to " - "the repository." - ), - "required": True, - }, - "nightly_build_from": { - "help": "Developement repository target", - }, - }, -) -def onedir( - ctx: Context, - salt_version: str = None, - incoming: pathlib.Path = None, - repo_path: pathlib.Path = None, - key_id: str = None, - nightly_build_from: str = None, -): - """ - Create the onedir repository. - """ - if TYPE_CHECKING: - assert salt_version is not None - assert incoming is not None - assert repo_path is not None - assert key_id is not None - _create_onedir_based_repo( - ctx, - salt_version=salt_version, - nightly_build_from=nightly_build_from, - repo_path=repo_path, - incoming=incoming, - key_id=key_id, - distro="onedir", - pkg_suffixes=(".xz", ".zip"), - ) - ctx.info("Done") - - -@create.command( - name="src", - arguments={ - "salt_version": { - "help": "The salt version for which to build the repository", - "required": True, - }, - "repo_path": { - "help": "Path where the repository shall be created.", - "required": True, - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "incoming": { - "help": ( - "The path to the directory containing the files that should added to " - "the repository." - ), - "required": True, - }, - "nightly_build_from": { - "help": "Developement repository target", - }, - }, -) -def src( - ctx: Context, - salt_version: str = None, - incoming: pathlib.Path = None, - repo_path: pathlib.Path = None, - key_id: str = None, - nightly_build_from: str = None, -): - """ - Create the onedir repository. - """ - if TYPE_CHECKING: - assert salt_version is not None - assert incoming is not None - assert repo_path is not None - assert key_id is not None - - ctx.info("Creating repository directory structure ...") - create_repo_path = _create_top_level_repo_path( - ctx, - repo_path, - salt_version, - distro="src", - nightly_build_from=nightly_build_from, - ) - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - create_repo_path = create_repo_path / salt_version - create_repo_path.mkdir(exist_ok=True, parents=True) - hashes_base_path = create_repo_path / f"salt-{salt_version}" - for fpath in incoming.iterdir(): - if fpath.suffix not in (".gz",): - continue - ctx.info(f"* Processing {fpath} ...") - dpath = create_repo_path / fpath.name - ctx.info(f"Copying {fpath} to {dpath} ...") - shutil.copyfile(fpath, dpath) - for hash_name in ("blake2b", "sha512", "sha3_512"): - ctx.info(f" * Calculating {hash_name} ...") - hexdigest = _get_file_checksum(fpath, hash_name) - with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh: - wfh.write(f"{hexdigest} {dpath.name}\n") - with open(f"{dpath}.{hash_name}", "a+") as wfh: - wfh.write(f"{hexdigest} {dpath.name}\n") - - for fpath in create_repo_path.iterdir(): - if fpath.suffix in (".pub", ".gpg"): - continue - tools.utils.gpg_sign(ctx, key_id, fpath) - - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - ctx.info("Done") - - -@publish.command( - arguments={ - "repo_path": { - "help": "Local path for the repository that shall be published.", - }, - "salt_version": { - "help": "The salt version for which to build the repository", - "required": True, - }, - } -) -def nightly(ctx: Context, repo_path: pathlib.Path, salt_version: str = None): - """ - Publish to the nightly bucket. - """ - if TYPE_CHECKING: - assert salt_version is not None - _publish_repo( - ctx, repo_path=repo_path, nightly_build=True, salt_version=salt_version - ) - - -@publish.command( - arguments={ - "repo_path": { - "help": "Local path for the repository that shall be published.", - }, - "salt_version": { - "help": "The salt version for which to build the repository", - "required": True, - }, - } -) -def staging(ctx: Context, repo_path: pathlib.Path, salt_version: str = None): - """ - Publish to the staging bucket. - """ - if TYPE_CHECKING: - assert salt_version is not None - _publish_repo(ctx, repo_path=repo_path, stage=True, salt_version=salt_version) - - -@repo.command(name="backup-previous-releases") -def backup_previous_releases(ctx: Context): - """ - Backup release bucket. - """ - _rclone(ctx, tools.utils.RELEASE_BUCKET_NAME, tools.utils.BACKUP_BUCKET_NAME) - ctx.info("Done") - - -@repo.command(name="restore-previous-releases") -def restore_previous_releases(ctx: Context): - """ - Restore release bucket from backup. - """ - _rclone(ctx, tools.utils.BACKUP_BUCKET_NAME, tools.utils.RELEASE_BUCKET_NAME) - github_output = os.environ.get("GITHUB_OUTPUT") - if github_output is not None: - with open(github_output, "a", encoding="utf-8") as wfh: - wfh.write(f"backup-complete=true\n") - ctx.info("Done") - - -def _rclone(ctx: Context, src: str, dst: str): - rclone = shutil.which("rclone") - if not rclone: - ctx.error("Could not find the rclone binary") - ctx.exit(1) - - if TYPE_CHECKING: - assert rclone - - env = os.environ.copy() - env["RCLONE_CONFIG_S3_TYPE"] = "s3" - cmdline: list[str] = [ - rclone, - "sync", - "--auto-confirm", - "--human-readable", - "--checksum", - "--color=always", - "--metadata", - "--s3-env-auth", - "--s3-location-constraint=us-west-2", - "--s3-provider=AWS", - "--s3-region=us-west-2", - "--stats-file-name-length=0", - "--stats-one-line", - "--stats=5s", - "--transfers=50", - "--fast-list", - "--verbose", - ] - if src == tools.utils.RELEASE_BUCKET_NAME: - cmdline.append("--s3-storage-class=INTELLIGENT_TIERING") - cmdline.extend([f"s3://{src}", f"s3://{dst}"]) - ctx.info(f"Running: {' '.join(cmdline)}") - ret = ctx.run(*cmdline, env=env, check=False) - if ret.returncode: - ctx.error(f"Failed to sync from s3://{src} to s3://{dst}") - ctx.exit(1) - - -@publish.command( - arguments={ - "salt_version": { - "help": "The salt version to release.", - }, - } -) -def release(ctx: Context, salt_version: str): - """ - Publish to the release bucket. - """ - if "rc" in salt_version: - bucket_folder = "salt_rc/salt/py3" - else: - bucket_folder = "salt/py3" - - files_to_copy: list[str] - directories_to_delete: list[str] = [] - - ctx.info("Grabbing remote file listing of files to copy...") - s3 = boto3.client("s3") - repo_release_files_path = pathlib.Path( - f"release-artifacts/{salt_version}/.release-files.json" - ) - repo_release_symlinks_path = pathlib.Path( - f"release-artifacts/{salt_version}/.release-symlinks.json" - ) - with tempfile.TemporaryDirectory(prefix=f"{salt_version}_release_") as tsd: - local_release_files_path = pathlib.Path(tsd) / repo_release_files_path.name - try: - bucket_name = tools.utils.STAGING_BUCKET_NAME - with local_release_files_path.open("wb") as wfh: - ctx.info( - f"Downloading {repo_release_files_path} from bucket {bucket_name} ..." - ) - s3.download_fileobj( - Bucket=bucket_name, - Key=str(repo_release_files_path), - Fileobj=wfh, - ) - files_to_copy = json.loads(local_release_files_path.read_text()) - except ClientError as exc: - if "Error" not in exc.response: - log.exception(f"Error downloading {repo_release_files_path}: {exc}") - ctx.exit(1) - if exc.response["Error"]["Code"] == "404": - ctx.error(f"Could not find {repo_release_files_path} in bucket.") - ctx.exit(1) - if exc.response["Error"]["Code"] == "400": - ctx.error( - f"Could not download {repo_release_files_path} from bucket: {exc}" - ) - ctx.exit(1) - log.exception(f"Error downloading {repo_release_files_path}: {exc}") - ctx.exit(1) - local_release_symlinks_path = ( - pathlib.Path(tsd) / repo_release_symlinks_path.name - ) - try: - with local_release_symlinks_path.open("wb") as wfh: - ctx.info( - f"Downloading {repo_release_symlinks_path} from bucket {bucket_name} ..." - ) - s3.download_fileobj( - Bucket=bucket_name, - Key=str(repo_release_symlinks_path), - Fileobj=wfh, - ) - directories_to_delete = json.loads(local_release_symlinks_path.read_text()) - except ClientError as exc: - if "Error" not in exc.response: - log.exception(f"Error downloading {repo_release_symlinks_path}: {exc}") - ctx.exit(1) - if exc.response["Error"]["Code"] == "404": - ctx.error(f"Could not find {repo_release_symlinks_path} in bucket.") - ctx.exit(1) - if exc.response["Error"]["Code"] == "400": - ctx.error( - f"Could not download {repo_release_symlinks_path} from bucket: {exc}" - ) - ctx.exit(1) - log.exception(f"Error downloading {repo_release_symlinks_path}: {exc}") - ctx.exit(1) - - if directories_to_delete: - with tools.utils.create_progress_bar() as progress: - task = progress.add_task( - "Deleting directories to override.", - total=len(directories_to_delete), - ) - for directory in directories_to_delete: - try: - objects_to_delete: list[dict[str, str]] = [] - for path in _get_repo_file_list( - bucket_name=tools.utils.RELEASE_BUCKET_NAME, - bucket_folder=bucket_folder, - glob_match=f"{directory}/**", - ): - objects_to_delete.append({"Key": path}) - if objects_to_delete: - s3.delete_objects( - Bucket=tools.utils.RELEASE_BUCKET_NAME, - Delete={"Objects": objects_to_delete}, - ) - except ClientError: - log.exception("Failed to delete remote files") - finally: - progress.update(task, advance=1) - - already_copied_files: list[str] = [] - s3 = boto3.client("s3") - dot_repo_files = [] - with tools.utils.create_progress_bar() as progress: - task = progress.add_task( - "Copying files between buckets", total=len(files_to_copy) - ) - for fpath in files_to_copy: - if fpath in already_copied_files: - continue - if fpath.endswith(".repo"): - dot_repo_files.append(fpath) - ctx.info(f" * Copying {fpath}") - try: - s3.copy_object( - Bucket=tools.utils.RELEASE_BUCKET_NAME, - Key=fpath, - CopySource={ - "Bucket": tools.utils.STAGING_BUCKET_NAME, - "Key": fpath, - }, - MetadataDirective="COPY", - TaggingDirective="COPY", - ServerSideEncryption="AES256", - ) - already_copied_files.append(fpath) - except ClientError: - log.exception(f"Failed to copy {fpath}") - finally: - progress.update(task, advance=1) - - # Now let's get the onedir based repositories where we need to update several repo.json - major_version = packaging.version.parse(salt_version).major - with tempfile.TemporaryDirectory(prefix=f"{salt_version}_release_") as tsd: - repo_path = pathlib.Path(tsd) - for distro in ("windows", "macos", "onedir"): - - create_repo_path = _create_repo_path( - ctx, - repo_path, - salt_version, - distro=distro, - ) - repo_json_path = create_repo_path.parent.parent / "repo.json" - - release_repo_json = _get_repo_json_file_contents( - ctx, - bucket_name=tools.utils.RELEASE_BUCKET_NAME, - repo_path=repo_path, - repo_json_path=repo_json_path, - ) - minor_repo_json_path = create_repo_path.parent / "repo.json" - - staging_minor_repo_json = _get_repo_json_file_contents( - ctx, - bucket_name=tools.utils.STAGING_BUCKET_NAME, - repo_path=repo_path, - repo_json_path=minor_repo_json_path, - ) - release_minor_repo_json = _get_repo_json_file_contents( - ctx, - bucket_name=tools.utils.RELEASE_BUCKET_NAME, - repo_path=repo_path, - repo_json_path=minor_repo_json_path, - ) - - release_json = staging_minor_repo_json[salt_version] - - major_version = Version(salt_version).major - versions = _parse_versions(*list(release_minor_repo_json)) - ctx.info( - f"Collected versions from {minor_repo_json_path.relative_to(repo_path)}: " - f"{', '.join(str(vs) for vs in versions)}" - ) - minor_versions = [v for v in versions if v.major == major_version] - ctx.info( - f"Collected versions(Matching major: {major_version}) from " - f"{minor_repo_json_path.relative_to(repo_path)}: " - f"{', '.join(str(vs) for vs in minor_versions)}" - ) - if not versions: - latest_version = Version(salt_version) - else: - latest_version = versions[0] - if not minor_versions: - latest_minor_version = Version(salt_version) - else: - latest_minor_version = minor_versions[0] - - ctx.info(f"Release Version: {salt_version}") - ctx.info(f"Latest Repo Version: {latest_version}") - ctx.info(f"Latest Release Minor Version: {latest_minor_version}") - - # Add the minor version - release_minor_repo_json[salt_version] = release_json - - if latest_version <= salt_version: - release_repo_json["latest"] = release_json - - if latest_minor_version <= salt_version: - release_minor_repo_json["latest"] = release_json - - ctx.info(f"Writing {minor_repo_json_path} ...") - minor_repo_json_path.write_text( - json.dumps(release_minor_repo_json, sort_keys=True) - ) - ctx.info(f"Writing {repo_json_path} ...") - repo_json_path.write_text(json.dumps(release_repo_json, sort_keys=True)) - - # And now, let's get the several rpm "*.repo" files to update the base - # domain from staging to release - release_domain = os.environ.get( - "SALT_REPO_DOMAIN_RELEASE", "repo.saltproject.io" - ) - for path in dot_repo_files: - repo_file_path = repo_path.joinpath(path) - repo_file_path.parent.mkdir(exist_ok=True, parents=True) - bucket_name = tools.utils.STAGING_BUCKET_NAME - try: - ret = s3.head_object(Bucket=bucket_name, Key=path) - ctx.info( - f"Downloading existing '{repo_file_path.relative_to(repo_path)}' " - f"file from bucket {bucket_name}" - ) - size = ret["ContentLength"] - with repo_file_path.open("wb") as wfh: - with tools.utils.create_progress_bar( - file_progress=True - ) as progress: - task = progress.add_task( - description="Downloading...", total=size - ) - s3.download_fileobj( - Bucket=bucket_name, - Key=path, - Fileobj=wfh, - Callback=tools.utils.UpdateProgress(progress, task), - ) - updated_contents = re.sub( - r"^(baseurl|gpgkey)=https://([^/]+)/(.*)$", - rf"\1=https://{release_domain}/\3", - repo_file_path.read_text(), - flags=re.MULTILINE, - ) - ctx.info(f"Updated '{repo_file_path.relative_to(repo_path)}:") - ctx.print(updated_contents) - repo_file_path.write_text(updated_contents) - except ClientError as exc: - if "Error" not in exc.response: - raise - if exc.response["Error"]["Code"] != "404": - raise - ctx.info(f"Could not find {repo_file_path} in bucket {bucket_name}") - - for dirpath, dirnames, filenames in os.walk(repo_path, followlinks=True): - for path in filenames: - upload_path = pathlib.Path(dirpath, path) - relpath = upload_path.relative_to(repo_path) - size = upload_path.stat().st_size - ctx.info(f" {relpath}") - with tools.utils.create_progress_bar(file_progress=True) as progress: - task = progress.add_task(description="Uploading...", total=size) - s3.upload_file( - str(upload_path), - tools.utils.RELEASE_BUCKET_NAME, - str(relpath), - Callback=tools.utils.UpdateProgress(progress, task), - ) - - -@publish.command( - arguments={ - "salt_version": { - "help": "The salt version to release.", - }, - "key_id": { - "help": "The GnuPG key ID used to sign.", - "required": True, - }, - "repository": { - "help": ( - "The full repository name, ie, 'saltstack/salt' on GitHub " - "to run the checks against." - ) - }, - } -) -def github( - ctx: Context, - salt_version: str, - key_id: str = None, - repository: str = "saltstack/salt", -): - """ - Publish the release on GitHub releases. - """ - if TYPE_CHECKING: - assert key_id is not None - - s3 = boto3.client("s3") - - # Let's download the release artifacts stored in staging - artifacts_path = pathlib.Path.cwd() / "release-artifacts" - artifacts_path.mkdir(exist_ok=True) - release_artifacts_listing: dict[pathlib.Path, int] = {} - continuation_token = None - while True: - kwargs: dict[str, str] = {} - if continuation_token: - kwargs["ContinuationToken"] = continuation_token - ret = s3.list_objects_v2( - Bucket=tools.utils.STAGING_BUCKET_NAME, - Prefix=f"release-artifacts/{salt_version}", - FetchOwner=False, - **kwargs, - ) - contents = ret.pop("Contents", None) - if contents is None: - break - for entry in contents: - entry_path = pathlib.Path(entry["Key"]) - if entry_path.name.startswith("."): - continue - release_artifacts_listing[entry_path] = entry["Size"] - if not ret["IsTruncated"]: - break - continuation_token = ret["NextContinuationToken"] - - for entry_path, size in release_artifacts_listing.items(): - ctx.info(f" * {entry_path.name}") - local_path = artifacts_path / entry_path.name - with local_path.open("wb") as wfh: - with tools.utils.create_progress_bar(file_progress=True) as progress: - task = progress.add_task(description="Downloading...", total=size) - s3.download_fileobj( - Bucket=tools.utils.STAGING_BUCKET_NAME, - Key=str(entry_path), - Fileobj=wfh, - Callback=tools.utils.UpdateProgress(progress, task), - ) - - for artifact in artifacts_path.iterdir(): - if artifact.suffix in (".patch", ".asc", ".gpg", ".pub"): - continue - tools.utils.gpg_sign(ctx, key_id, artifact) - - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, artifacts_path) - - release_message = f"""\ - # Welcome to Salt v{salt_version} - - | :exclamation: ATTENTION | - |:-------------------------------------------------------------------------------------------------------------------------| - | The archives generated by GitHub(`Source code(zip)`, `Source code(tar.gz)`) will not report Salt's version properly. | - | Please use the tarball generated by The Salt Project Team(`salt-{salt_version}.tar.gz`). - """ - release_message_path = artifacts_path / "gh-release-body.md" - release_message_path.write_text(textwrap.dedent(release_message).strip()) - - github_output = os.environ.get("GITHUB_OUTPUT") - if github_output is None: - ctx.warn("The 'GITHUB_OUTPUT' variable is not set. Stop processing.") - ctx.exit(0) - - if TYPE_CHECKING: - assert github_output is not None - - with open(github_output, "a", encoding="utf-8") as wfh: - wfh.write(f"release-messsage-file={release_message_path.resolve()}\n") - - releases = get_salt_releases(ctx, repository) - if Version(salt_version) >= releases[-1]: - make_latest = True - else: - make_latest = False - with open(github_output, "a", encoding="utf-8") as wfh: - wfh.write(f"make-latest={json.dumps(make_latest)}\n") - - artifacts_to_upload = [] - for artifact in artifacts_path.iterdir(): - if artifact.suffix == ".patch": - continue - if artifact.name == release_message_path.name: - continue - artifacts_to_upload.append(str(artifact.resolve())) - - with open(github_output, "a", encoding="utf-8") as wfh: - wfh.write(f"release-artifacts={','.join(artifacts_to_upload)}\n") - ctx.exit(0) - - -@repo.command( - name="confirm-unreleased", - arguments={ - "salt_version": { - "help": "The salt version to check", - }, - "repository": { - "help": ( - "The full repository name, ie, 'saltstack/salt' on GitHub " - "to run the checks against." - ) - }, - }, -) -def confirm_unreleased( - ctx: Context, salt_version: str, repository: str = "saltstack/salt" -): - """ - Confirm that the passed version is not yet tagged and/or released. - """ - releases = get_salt_releases(ctx, repository) - if Version(salt_version) in releases: - ctx.error(f"There's already a '{salt_version}' tag or github release.") - ctx.exit(1) - ctx.info(f"Could not find a release for Salt Version '{salt_version}'") - ctx.exit(0) - - -@repo.command( - name="confirm-staged", - arguments={ - "salt_version": { - "help": "The salt version to check", - }, - "repository": { - "help": ( - "The full repository name, ie, 'saltstack/salt' on GitHub " - "to run the checks against." - ) - }, - }, -) -def confirm_staged(ctx: Context, salt_version: str, repository: str = "saltstack/salt"): - """ - Confirm that the passed version has been staged for release. - """ - s3 = boto3.client("s3") - repo_release_files_path = pathlib.Path( - f"release-artifacts/{salt_version}/.release-files.json" - ) - repo_release_symlinks_path = pathlib.Path( - f"release-artifacts/{salt_version}/.release-symlinks.json" - ) - for remote_path in (repo_release_files_path, repo_release_symlinks_path): - try: - bucket_name = tools.utils.STAGING_BUCKET_NAME - ctx.info( - f"Checking for the presence of {remote_path} on bucket {bucket_name} ..." - ) - s3.head_object( - Bucket=bucket_name, - Key=str(remote_path), - ) - except ClientError as exc: - if "Error" not in exc.response: - log.exception(f"Could not get information about {remote_path}: {exc}") - ctx.exit(1) - if exc.response["Error"]["Code"] == "404": - ctx.error(f"Could not find {remote_path} in bucket.") - ctx.exit(1) - if exc.response["Error"]["Code"] == "400": - ctx.error(f"Could get information about {remote_path}: {exc}") - ctx.exit(1) - log.exception(f"Error getting information about {remote_path}: {exc}") - ctx.exit(1) - ctx.info(f"Version {salt_version} has been staged for release") - ctx.exit(0) - - -def _get_repo_detailed_file_list( - bucket_name: str, - bucket_folder: str = "", - glob_match: str = "**", -) -> list[dict[str, Any]]: - s3 = boto3.client("s3") - listing: list[dict[str, Any]] = [] - continuation_token = None - while True: - kwargs: dict[str, str] = {} - if continuation_token: - kwargs["ContinuationToken"] = continuation_token - ret = s3.list_objects_v2( - Bucket=bucket_name, - Prefix=bucket_folder, - FetchOwner=False, - **kwargs, - ) - contents = ret.pop("Contents", None) - if contents is None: - break - for entry in contents: - if fnmatch.fnmatch(entry["Key"], glob_match): - listing.append(entry) - if not ret["IsTruncated"]: - break - continuation_token = ret["NextContinuationToken"] - return listing - - -def _get_repo_file_list( - bucket_name: str, bucket_folder: str, glob_match: str -) -> list[str]: - return [ - entry["Key"] - for entry in _get_repo_detailed_file_list( - bucket_name, bucket_folder, glob_match=glob_match - ) - ] - - -def _get_remote_versions(bucket_name: str, remote_path: str): - log.info( - "Getting remote versions from bucket %r under path: %s", - bucket_name, - remote_path, - ) - remote_path = str(remote_path) - if not remote_path.endswith("/"): - remote_path += "/" - - s3 = boto3.client("s3") - ret = s3.list_objects( - Bucket=bucket_name, - Delimiter="/", - Prefix=remote_path, - ) - if "CommonPrefixes" not in ret: - return [] - versions = [] - for entry in ret["CommonPrefixes"]: - _, version = entry["Prefix"].rstrip("/").rsplit("/", 1) - if version == "latest": - continue - versions.append(Version(version)) - versions.sort(reverse=True) - log.info("Remote versions collected: %s", versions) - return versions - - -def _create_onedir_based_repo( - ctx: Context, - salt_version: str, - nightly_build_from: str | None, - repo_path: pathlib.Path, - incoming: pathlib.Path, - key_id: str, - distro: str, - pkg_suffixes: tuple[str, ...], -): - ctx.info("Creating repository directory structure ...") - create_repo_path = _create_top_level_repo_path( - ctx, - repo_path, - salt_version, - distro, - nightly_build_from=nightly_build_from, - ) - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - - create_repo_path = _create_repo_path( - ctx, - repo_path, - salt_version, - distro, - nightly_build_from=nightly_build_from, - ) - if not nightly_build_from: - repo_json_path = create_repo_path.parent.parent / "repo.json" - else: - repo_json_path = create_repo_path.parent / "repo.json" - - if nightly_build_from: - bucket_name = tools.utils.RELEASE_BUCKET_NAME - else: - bucket_name = tools.utils.STAGING_BUCKET_NAME - - release_json = {} - - copy_exclusions = ( - ".blake2b", - ".sha512", - ".sha3_512", - ".BLAKE2B", - ".SHA512", - ".SHA3_512", - ".json", - ) - hashes_base_path = create_repo_path / f"salt-{salt_version}" - for fpath in incoming.iterdir(): - if fpath.suffix in copy_exclusions: - continue - ctx.info(f"* Processing {fpath} ...") - dpath = create_repo_path / fpath.name - ctx.info(f"Copying {fpath} to {dpath} ...") - shutil.copyfile(fpath, dpath) - if "-amd64" in dpath.name.lower(): - arch = "amd64" - elif "-x86_64" in dpath.name.lower(): - arch = "x86_64" - elif "-x86" in dpath.name.lower(): - arch = "x86" - elif "-aarch64" in dpath.name.lower(): - arch = "aarch64" - else: - ctx.error( - f"Cannot pickup the right architecture from the filename '{dpath.name}'." - ) - ctx.exit(1) - if distro == "onedir": - if "-onedir-linux-" in dpath.name.lower(): - release_os = "linux" - elif "-onedir-darwin-" in dpath.name.lower(): - release_os = "macos" - elif "-onedir-windows-" in dpath.name.lower(): - release_os = "windows" - else: - ctx.error( - f"Cannot pickup the right OS from the filename '{dpath.name}'." - ) - ctx.exit(1) - else: - release_os = distro - release_json[dpath.name] = { - "name": dpath.name, - "version": salt_version, - "os": release_os, - "arch": arch, - } - for hash_name in ("blake2b", "sha512", "sha3_512"): - ctx.info(f" * Calculating {hash_name} ...") - hexdigest = _get_file_checksum(fpath, hash_name) - release_json[dpath.name][hash_name.upper()] = hexdigest - with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh: - wfh.write(f"{hexdigest} {dpath.name}\n") - with open(f"{dpath}.{hash_name}", "a+") as wfh: - wfh.write(f"{hexdigest} {dpath.name}\n") - - for fpath in create_repo_path.iterdir(): - if fpath.suffix in pkg_suffixes: - continue - tools.utils.gpg_sign(ctx, key_id, fpath) - - # Export the GPG key in use - tools.utils.export_gpg_key(ctx, key_id, create_repo_path) - - repo_json = _get_repo_json_file_contents( - ctx, bucket_name=bucket_name, repo_path=repo_path, repo_json_path=repo_json_path - ) - if nightly_build_from: - ctx.info(f"Writing {repo_json_path} ...") - repo_json_path.write_text(json.dumps(repo_json, sort_keys=True)) - return - - major_version = Version(salt_version).major - minor_repo_json_path = create_repo_path.parent / "repo.json" - minor_repo_json = _get_repo_json_file_contents( - ctx, - bucket_name=bucket_name, - repo_path=repo_path, - repo_json_path=minor_repo_json_path, - ) - minor_repo_json[salt_version] = release_json - versions = _parse_versions(*list(minor_repo_json)) - ctx.info( - f"Collected versions from {minor_repo_json_path.relative_to(repo_path)}: " - f"{', '.join(str(vs) for vs in versions)}" - ) - minor_versions = [v for v in versions if v.major == major_version] - ctx.info( - f"Collected versions(Matching major: {major_version}) from " - f"{minor_repo_json_path.relative_to(repo_path)}: " - f"{', '.join(str(vs) for vs in minor_versions)}" - ) - if not versions: - latest_version = Version(salt_version) - else: - latest_version = versions[0] - if not minor_versions: - latest_minor_version = Version(salt_version) - else: - latest_minor_version = minor_versions[0] - - ctx.info(f"Release Version: {salt_version}") - ctx.info(f"Latest Repo Version: {latest_version}") - ctx.info(f"Latest Release Minor Version: {latest_minor_version}") - - latest_link = create_repo_path.parent.parent / "latest" - if latest_version <= salt_version: - repo_json["latest"] = release_json - ctx.info(f"Creating '{latest_link.relative_to(repo_path)}' symlink ...") - if latest_link.exists(): - latest_link.unlink() - latest_link.symlink_to(f"minor/{salt_version}") - else: - ctx.info( - f"Not creating the '{latest_link.relative_to(repo_path)}' symlink " - f"since {latest_version} > {salt_version}" - ) - - major_link = create_repo_path.parent.parent / str(major_version) - if latest_minor_version <= salt_version: - minor_repo_json["latest"] = release_json - # This is the latest minor, update the major in the top level repo.json - # to this version - repo_json[str(major_version)] = release_json - ctx.info(f"Creating '{major_link.relative_to(repo_path)}' symlink ...") - if major_link.exists(): - major_link.unlink() - major_link.symlink_to(f"minor/{salt_version}") - else: - ctx.info( - f"Not creating the '{major_link.relative_to(repo_path)}' symlink " - f"since {latest_minor_version} > {salt_version}" - ) - - ctx.info(f"Writing {minor_repo_json_path} ...") - minor_repo_json_path.write_text(json.dumps(minor_repo_json, sort_keys=True)) - - ctx.info(f"Writing {repo_json_path} ...") - repo_json_path.write_text(json.dumps(repo_json, sort_keys=True)) - - -def _get_repo_json_file_contents( - ctx: Context, - bucket_name: str, - repo_path: pathlib.Path, - repo_json_path: pathlib.Path, -) -> dict[str, Any]: - s3 = boto3.client("s3") - repo_json: dict[str, Any] = {} - try: - ret = s3.head_object( - Bucket=bucket_name, Key=str(repo_json_path.relative_to(repo_path)) - ) - ctx.info( - f"Downloading existing '{repo_json_path.relative_to(repo_path)}' file " - f"from bucket {bucket_name}" - ) - size = ret["ContentLength"] - with repo_json_path.open("wb") as wfh: - with tools.utils.create_progress_bar(file_progress=True) as progress: - task = progress.add_task(description="Downloading...", total=size) - s3.download_fileobj( - Bucket=bucket_name, - Key=str(repo_json_path.relative_to(repo_path)), - Fileobj=wfh, - Callback=tools.utils.UpdateProgress(progress, task), - ) - with repo_json_path.open() as rfh: - repo_json = json.load(rfh) - except ClientError as exc: - if "Error" not in exc.response: - raise - if exc.response["Error"]["Code"] != "404": - raise - ctx.info(f"Could not find {repo_json_path} in bucket {bucket_name}") - if repo_json: - ctx.print(repo_json, soft_wrap=True) - return repo_json - - -def _get_file_checksum(fpath: pathlib.Path, hash_name: str) -> str: - - with fpath.open("rb") as rfh: - try: - digest = hashlib.file_digest(rfh, hash_name) # type: ignore[attr-defined] - except AttributeError: - # Python < 3.11 - buf = bytearray(2**18) # Reusable buffer to reduce allocations. - view = memoryview(buf) - digest = getattr(hashlib, hash_name)() - while True: - size = rfh.readinto(buf) - if size == 0: - break # EOF - digest.update(view[:size]) - hexdigest: str = digest.hexdigest() - return hexdigest - - -def _publish_repo( - ctx: Context, - repo_path: pathlib.Path, - salt_version: str, - nightly_build: bool = False, - stage: bool = False, -): - """ - Publish packaging repositories. - """ - if nightly_build: - bucket_name = tools.utils.RELEASE_BUCKET_NAME - elif stage: - bucket_name = tools.utils.STAGING_BUCKET_NAME - else: - bucket_name = tools.utils.RELEASE_BUCKET_NAME - - ctx.info("Preparing upload ...") - s3 = boto3.client("s3") - to_delete_paths: dict[pathlib.Path, list[dict[str, str]]] = {} - to_upload_paths: list[pathlib.Path] = [] - symlink_paths: list[str] = [] - uploaded_files: list[str] = [] - for dirpath, dirnames, filenames in os.walk(repo_path, followlinks=True): - for dirname in dirnames: - path = pathlib.Path(dirpath, dirname) - if not path.is_symlink(): - continue - # This is a symlink, then we need to delete all files under - # that directory in S3 because S3 does not understand symlinks - # and we would end up adding files to that folder instead of - # replacing it. - try: - relpath = path.relative_to(repo_path) - ret = s3.list_objects( - Bucket=bucket_name, - Prefix=str(relpath), - ) - if "Contents" not in ret: - continue - objects = [] - for entry in ret["Contents"]: - objects.append({"Key": entry["Key"]}) - to_delete_paths[path] = objects - symlink_paths.append(str(relpath)) - except ClientError as exc: - if "Error" not in exc.response: - raise - if exc.response["Error"]["Code"] != "404": - raise - - for fpath in filenames: - path = pathlib.Path(dirpath, fpath) - to_upload_paths.append(path) - - with tools.utils.create_progress_bar() as progress: - task = progress.add_task( - "Deleting directories to override.", total=len(to_delete_paths) - ) - for base, objects in to_delete_paths.items(): - relpath = base.relative_to(repo_path) - bucket_uri = f"s3://{bucket_name}/{relpath}" - progress.update(task, description=f"Deleting {bucket_uri}") - try: - ret = s3.delete_objects( - Bucket=bucket_name, - Delete={"Objects": objects}, - ) - except ClientError: - log.exception(f"Failed to delete {bucket_uri}") - finally: - progress.update(task, advance=1) - - try: - ctx.info("Uploading repository ...") - for upload_path in to_upload_paths: - relpath = upload_path.relative_to(repo_path) - size = upload_path.stat().st_size - ctx.info(f" {relpath}") - with tools.utils.create_progress_bar(file_progress=True) as progress: - task = progress.add_task(description="Uploading...", total=size) - s3.upload_file( - str(upload_path), - bucket_name, - str(relpath), - Callback=tools.utils.UpdateProgress(progress, task), - ExtraArgs={ - "Metadata": { - "x-amz-meta-salt-release-version": salt_version, - } - }, - ) - uploaded_files.append(str(relpath)) - if stage is True: - repo_files_path = f"release-artifacts/{salt_version}/.release-files.json" - ctx.info(f"Uploading {repo_files_path} ...") - s3.put_object( - Key=repo_files_path, - Bucket=bucket_name, - Body=json.dumps(uploaded_files).encode(), - Metadata={ - "x-amz-meta-salt-release-version": salt_version, - }, - ) - repo_symlinks_path = ( - f"release-artifacts/{salt_version}/.release-symlinks.json" - ) - ctx.info(f"Uploading {repo_symlinks_path} ...") - s3.put_object( - Key=repo_symlinks_path, - Bucket=bucket_name, - Body=json.dumps(symlink_paths).encode(), - Metadata={ - "x-amz-meta-salt-release-version": salt_version, - }, - ) - except KeyboardInterrupt: - pass - - -def _create_top_level_repo_path( - ctx: Context, - repo_path: pathlib.Path, - salt_version: str, - distro: str, - distro_version: str | None = None, # pylint: disable=bad-whitespace - distro_arch: str | None = None, # pylint: disable=bad-whitespace - nightly_build_from: str | None = None, # pylint: disable=bad-whitespace -): - create_repo_path = repo_path - if nightly_build_from: - create_repo_path = ( - create_repo_path - / "salt-dev" - / nightly_build_from - / datetime.utcnow().strftime("%Y-%m-%d") - ) - create_repo_path.mkdir(exist_ok=True, parents=True) - with ctx.chdir(create_repo_path.parent): - latest_nightly_symlink = pathlib.Path("latest") - if not latest_nightly_symlink.exists(): - ctx.info( - f"Creating 'latest' symlink to '{create_repo_path.relative_to(repo_path)}' ..." - ) - latest_nightly_symlink.symlink_to( - create_repo_path.name, target_is_directory=True - ) - elif "rc" in salt_version: - create_repo_path = create_repo_path / "salt_rc" - create_repo_path = create_repo_path / "salt" / "py3" / distro - if distro_version: - create_repo_path = create_repo_path / distro_version - if distro_arch: - create_repo_path = create_repo_path / distro_arch - create_repo_path.mkdir(exist_ok=True, parents=True) - return create_repo_path - - -def _create_repo_path( - ctx: Context, - repo_path: pathlib.Path, - salt_version: str, - distro: str, - distro_version: str | None = None, # pylint: disable=bad-whitespace - distro_arch: str | None = None, # pylint: disable=bad-whitespace - nightly_build_from: str | None = None, # pylint: disable=bad-whitespace -): - create_repo_path = _create_top_level_repo_path( - ctx, - repo_path, - salt_version, - distro, - distro_version, - distro_arch, - nightly_build_from=nightly_build_from, - ) - create_repo_path = create_repo_path / "minor" / salt_version - create_repo_path.mkdir(exist_ok=True, parents=True) - return create_repo_path - - -def _parse_versions(*versions: str) -> list[Version]: - _versions = [] - for version in set(versions): - if version == "latest": - continue - _versions.append(Version(version)) - if _versions: - _versions.sort(reverse=True) - return _versions diff --git a/tools/pkg/repo/__init__.py b/tools/pkg/repo/__init__.py new file mode 100644 index 00000000000..8a3cbd9c81f --- /dev/null +++ b/tools/pkg/repo/__init__.py @@ -0,0 +1,181 @@ +""" +These commands are used to build the pacakge repository files. +""" +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import logging +import os +import pathlib +import shutil +import sys +from typing import TYPE_CHECKING + +from ptscripts import Context, command_group + +import tools.pkg +import tools.utils +from tools.utils import Version, get_salt_releases + +try: + import boto3 + from botocore.exceptions import ClientError +except ImportError: + print( + "\nPlease run 'python -m pip install -r " + "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info), + file=sys.stderr, + flush=True, + ) + raise + +log = logging.getLogger(__name__) + +# Define the command group +repo = command_group( + name="repo", + help="Packaging Repository Related Commands", + description=__doc__, + parent="pkg", +) + + +@repo.command(name="backup-previous-releases") +def backup_previous_releases(ctx: Context): + """ + Backup release bucket. + """ + _rclone(ctx, tools.utils.RELEASE_BUCKET_NAME, tools.utils.BACKUP_BUCKET_NAME) + ctx.info("Done") + + +@repo.command(name="restore-previous-releases") +def restore_previous_releases(ctx: Context): + """ + Restore release bucket from backup. + """ + _rclone(ctx, tools.utils.BACKUP_BUCKET_NAME, tools.utils.RELEASE_BUCKET_NAME) + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output is not None: + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"backup-complete=true\n") + ctx.info("Done") + + +def _rclone(ctx: Context, src: str, dst: str): + rclone = shutil.which("rclone") + if not rclone: + ctx.error("Could not find the rclone binary") + ctx.exit(1) + + if TYPE_CHECKING: + assert rclone + + env = os.environ.copy() + env["RCLONE_CONFIG_S3_TYPE"] = "s3" + cmdline: list[str] = [ + rclone, + "sync", + "--auto-confirm", + "--human-readable", + "--checksum", + "--color=always", + "--metadata", + "--s3-env-auth", + "--s3-location-constraint=us-west-2", + "--s3-provider=AWS", + "--s3-region=us-west-2", + "--stats-file-name-length=0", + "--stats-one-line", + "--stats=5s", + "--transfers=50", + "--fast-list", + "--verbose", + ] + if src == tools.utils.RELEASE_BUCKET_NAME: + cmdline.append("--s3-storage-class=INTELLIGENT_TIERING") + cmdline.extend([f"s3://{src}", f"s3://{dst}"]) + ctx.info(f"Running: {' '.join(cmdline)}") + ret = ctx.run(*cmdline, env=env, check=False) + if ret.returncode: + ctx.error(f"Failed to sync from s3://{src} to s3://{dst}") + ctx.exit(1) + + +@repo.command( + name="confirm-unreleased", + arguments={ + "salt_version": { + "help": "The salt version to check", + }, + "repository": { + "help": ( + "The full repository name, ie, 'saltstack/salt' on GitHub " + "to run the checks against." + ) + }, + }, +) +def confirm_unreleased( + ctx: Context, salt_version: str, repository: str = "saltstack/salt" +): + """ + Confirm that the passed version is not yet tagged and/or released. + """ + releases = get_salt_releases(ctx, repository) + if Version(salt_version) in releases: + ctx.error(f"There's already a '{salt_version}' tag or github release.") + ctx.exit(1) + ctx.info(f"Could not find a release for Salt Version '{salt_version}'") + ctx.exit(0) + + +@repo.command( + name="confirm-staged", + arguments={ + "salt_version": { + "help": "The salt version to check", + }, + "repository": { + "help": ( + "The full repository name, ie, 'saltstack/salt' on GitHub " + "to run the checks against." + ) + }, + }, +) +def confirm_staged(ctx: Context, salt_version: str, repository: str = "saltstack/salt"): + """ + Confirm that the passed version has been staged for release. + """ + s3 = boto3.client("s3") + repo_release_files_path = pathlib.Path( + f"release-artifacts/{salt_version}/.release-files.json" + ) + repo_release_symlinks_path = pathlib.Path( + f"release-artifacts/{salt_version}/.release-symlinks.json" + ) + for remote_path in (repo_release_files_path, repo_release_symlinks_path): + try: + bucket_name = tools.utils.STAGING_BUCKET_NAME + ctx.info( + f"Checking for the presence of {remote_path} on bucket {bucket_name} ..." + ) + s3.head_object( + Bucket=bucket_name, + Key=str(remote_path), + ) + except ClientError as exc: + if "Error" not in exc.response: + log.exception(f"Could not get information about {remote_path}: {exc}") + ctx.exit(1) + if exc.response["Error"]["Code"] == "404": + ctx.error(f"Could not find {remote_path} in bucket.") + ctx.exit(1) + if exc.response["Error"]["Code"] == "400": + ctx.error(f"Could get information about {remote_path}: {exc}") + ctx.exit(1) + log.exception(f"Error getting information about {remote_path}: {exc}") + ctx.exit(1) + ctx.info(f"Version {salt_version} has been staged for release") + ctx.exit(0) diff --git a/tools/pkg/repo/create.py b/tools/pkg/repo/create.py new file mode 100644 index 00000000000..ec4b3331c42 --- /dev/null +++ b/tools/pkg/repo/create.py @@ -0,0 +1,1038 @@ +""" +These commands are used to build the pacakge repository files. +""" +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import hashlib +import json +import logging +import os +import pathlib +import shutil +import sys +import textwrap +from typing import TYPE_CHECKING + +from ptscripts import Context, command_group + +import tools.pkg +import tools.utils +from tools.utils import ( + Version, + create_full_repo_path, + create_top_level_repo_path, + get_repo_json_file_contents, + parse_versions, +) + +try: + import boto3 +except ImportError: + print( + "\nPlease run 'python -m pip install -r " + "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info), + file=sys.stderr, + flush=True, + ) + raise + +log = logging.getLogger(__name__) + +create = command_group( + name="create", + help="Packaging Repository Creation Related Commands", + parent=["pkg", "repo"], +) + + +_deb_distro_info = { + "debian": { + "10": { + "label": "deb10ary", + "codename": "buster", + "suitename": "oldstable", + }, + "11": { + "label": "deb11ary", + "codename": "bullseye", + "suitename": "stable", + }, + }, + "ubuntu": { + "20.04": { + "label": "salt_ubuntu2004", + "codename": "focal", + }, + "22.04": { + "label": "salt_ubuntu2204", + "codename": "jammy", + }, + }, +} + + +@create.command( + name="deb", + arguments={ + "salt_version": { + "help": ( + "The salt version for which to build the repository configuration files. " + "If not passed, it will be discovered by running 'python3 salt/version.py'." + ), + "required": True, + }, + "distro": { + "help": "The debian based distribution to build the repository for", + "choices": list(_deb_distro_info), + "required": True, + }, + "distro_version": { + "help": "The distro version.", + "required": True, + }, + "distro_arch": { + "help": "The distribution architecture", + "choices": ("x86_64", "amd64", "aarch64", "arm64"), + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + "nightly_build_from": { + "help": "Developement repository target", + }, + }, +) +def debian( + ctx: Context, + salt_version: str = None, + distro: str = None, + distro_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + distro_arch: str = "amd64", + nightly_build_from: str = None, +): + """ + Create the debian repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert distro is not None + assert distro_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + display_name = f"{distro.capitalize()} {distro_version}" + if distro_version not in _deb_distro_info[distro]: + ctx.error(f"Support for {display_name} is missing.") + ctx.exit(1) + + if distro_arch == "x86_64": + ctx.info(f"The {distro_arch} arch is an alias for 'amd64'. Adjusting.") + distro_arch = "amd64" + + if distro_arch == "aarch64": + ctx.info(f"The {distro_arch} arch is an alias for 'arm64'. Adjusting.") + distro_arch = "arm64" + + distro_details = _deb_distro_info[distro][distro_version] + + ctx.info("Distribution Details:") + ctx.info(distro_details) + if TYPE_CHECKING: + assert isinstance(distro_details["label"], str) + assert isinstance(distro_details["codename"], str) + assert isinstance(distro_details["suitename"], str) + label: str = distro_details["label"] + codename: str = distro_details["codename"] + + ftp_archive_config_suite = "" + if distro == "debian": + suitename: str = distro_details["suitename"] + ftp_archive_config_suite = ( + f"""\n APT::FTPArchive::Release::Suite "{suitename}";\n""" + ) + archive_description = f"SaltProject {display_name} Python 3{'' if not nightly_build_from else ' development'} Salt package repo" + ftp_archive_config = f"""\ + APT::FTPArchive::Release::Origin "SaltProject"; + APT::FTPArchive::Release::Label "{label}";{ftp_archive_config_suite} + APT::FTPArchive::Release::Codename "{codename}"; + APT::FTPArchive::Release::Architectures "{distro_arch}"; + APT::FTPArchive::Release::Components "main"; + APT::FTPArchive::Release::Description "{archive_description}"; + APT::FTPArchive::Release::Acquire-By-Hash "yes"; + Dir {{ + ArchiveDir "."; + }}; + BinDirectory "pool" {{ + Packages "dists/{codename}/main/binary-{distro_arch}/Packages"; + Sources "dists/{codename}/main/source/Sources"; + Contents "dists/{codename}/main/Contents-{distro_arch}"; + }} + """ + ctx.info("Creating repository directory structure ...") + create_repo_path = create_top_level_repo_path( + ctx, + repo_path, + salt_version, + distro, + distro_version=distro_version, + distro_arch=distro_arch, + nightly_build_from=nightly_build_from, + ) + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + + create_repo_path = create_full_repo_path( + ctx, + repo_path, + salt_version, + distro, + distro_version=distro_version, + distro_arch=distro_arch, + nightly_build_from=nightly_build_from, + ) + ftp_archive_config_file = create_repo_path / "apt-ftparchive.conf" + ctx.info(f"Writing {ftp_archive_config_file} ...") + ftp_archive_config_file.write_text(textwrap.dedent(ftp_archive_config)) + + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + + pool_path = create_repo_path / "pool" + pool_path.mkdir(exist_ok=True) + for fpath in incoming.iterdir(): + dpath = pool_path / fpath.name + ctx.info(f"Copying {fpath} to {dpath} ...") + shutil.copyfile(fpath, dpath) + if fpath.suffix == ".dsc": + ctx.info(f"Running 'debsign' on {dpath} ...") + ctx.run("debsign", "--re-sign", "-k", key_id, str(dpath), interactive=True) + + dists_path = create_repo_path / "dists" + symlink_parent_path = dists_path / codename / "main" + symlink_paths = ( + symlink_parent_path / "by-hash" / "SHA256", + symlink_parent_path / "source" / "by-hash" / "SHA256", + symlink_parent_path / f"binary-{distro_arch}" / "by-hash" / "SHA256", + ) + + for path in symlink_paths: + path.mkdir(exist_ok=True, parents=True) + + cmdline = ["apt-ftparchive", "generate", "apt-ftparchive.conf"] + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ctx.run(*cmdline, cwd=create_repo_path) + + ctx.info("Creating by-hash symlinks ...") + for path in symlink_paths: + for fpath in path.parent.parent.iterdir(): + if not fpath.is_file(): + continue + sha256sum = ctx.run("sha256sum", str(fpath), capture=True) + link = path / sha256sum.stdout.decode().split()[0] + link.symlink_to(f"../../{fpath.name}") + + cmdline = [ + "apt-ftparchive", + "--no-md5", + "--no-sha1", + "--no-sha512", + "release", + "-c", + "apt-ftparchive.conf", + f"dists/{codename}/", + ] + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ret = ctx.run(*cmdline, capture=True, cwd=create_repo_path) + release_file = dists_path / codename / "Release" + ctx.info(f"Writing {release_file} with the output of the previous command...") + release_file.write_bytes(ret.stdout) + + cmdline = [ + "gpg", + "-u", + key_id, + "-o", + f"dists/{codename}/InRelease", + "-a", + "-s", + "--clearsign", + f"dists/{codename}/Release", + ] + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ctx.run(*cmdline, cwd=create_repo_path) + + cmdline = [ + "gpg", + "-u", + key_id, + "-o", + f"dists/{codename}/Release.gpg", + "-a", + "-b", + "-s", + f"dists/{codename}/Release", + ] + + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ctx.run(*cmdline, cwd=create_repo_path) + if not nightly_build_from: + remote_versions = _get_remote_versions( + tools.utils.STAGING_BUCKET_NAME, + create_repo_path.parent.relative_to(repo_path), + ) + major_version = Version(salt_version).major + matching_major = None + for version in remote_versions: + if version.major == major_version: + matching_major = version + break + if not matching_major or matching_major <= salt_version: + major_link = create_repo_path.parent.parent / str(major_version) + ctx.info(f"Creating '{major_link.relative_to(repo_path)}' symlink ...") + major_link.symlink_to(f"minor/{salt_version}") + if not remote_versions or remote_versions[0] <= salt_version: + latest_link = create_repo_path.parent.parent / "latest" + ctx.info(f"Creating '{latest_link.relative_to(repo_path)}' symlink ...") + latest_link.symlink_to(f"minor/{salt_version}") + + ctx.info("Done") + + +_rpm_distro_info = { + "amazon": ["2"], + "redhat": ["7", "8", "9"], + "fedora": ["36", "37", "38"], + "photon": ["3", "4"], +} + + +@create.command( + name="rpm", + arguments={ + "salt_version": { + "help": ( + "The salt version for which to build the repository configuration files. " + "If not passed, it will be discovered by running 'python3 salt/version.py'." + ), + "required": True, + }, + "distro": { + "help": "The debian based distribution to build the repository for", + "choices": list(_rpm_distro_info), + "required": True, + }, + "distro_version": { + "help": "The distro version.", + "required": True, + }, + "distro_arch": { + "help": "The distribution architecture", + "choices": ("x86_64", "aarch64", "arm64"), + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + "nightly_build_from": { + "help": "Developement repository target", + }, + }, +) +def rpm( + ctx: Context, + salt_version: str = None, + distro: str = None, + distro_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + distro_arch: str = "amd64", + nightly_build_from: str = None, +): + """ + Create the redhat repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert distro is not None + assert distro_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + display_name = f"{distro.capitalize()} {distro_version}" + if distro_version not in _rpm_distro_info[distro]: + ctx.error(f"Support for {display_name} is missing.") + ctx.exit(1) + + if distro_arch == "aarch64": + ctx.info(f"The {distro_arch} arch is an alias for 'arm64'. Adjusting.") + distro_arch = "arm64" + + ctx.info("Creating repository directory structure ...") + create_repo_path = create_top_level_repo_path( + ctx, + repo_path, + salt_version, + distro, + distro_version=distro_version, + distro_arch=distro_arch, + nightly_build_from=nightly_build_from, + ) + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + + create_repo_path = create_full_repo_path( + ctx, + repo_path, + salt_version, + distro, + distro_version=distro_version, + distro_arch=distro_arch, + nightly_build_from=nightly_build_from, + ) + + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + + for fpath in incoming.iterdir(): + if ".src" in fpath.suffixes: + dpath = create_repo_path / "SRPMS" / fpath.name + else: + dpath = create_repo_path / fpath.name + ctx.info(f"Copying {fpath} to {dpath} ...") + shutil.copyfile(fpath, dpath) + if fpath.suffix == ".rpm": + ctx.info(f"Running 'rpmsign' on {dpath} ...") + ctx.run( + "rpmsign", + "--key-id", + key_id, + "--addsign", + "--digest-algo=sha256", + str(dpath), + ) + + createrepo = shutil.which("createrepo") + if createrepo is None: + container = "ghcr.io/saltstack/salt-ci-containers/packaging:centosstream-9" + ctx.info(f"Using docker container '{container}' to call 'createrepo'...") + uid = ctx.run("id", "-u", capture=True).stdout.strip().decode() + gid = ctx.run("id", "-g", capture=True).stdout.strip().decode() + ctx.run( + "docker", + "run", + "--rm", + "-v", + f"{create_repo_path.resolve()}:/code", + "-u", + f"{uid}:{gid}", + "-w", + "/code", + container, + "createrepo", + ".", + ) + else: + ctx.run("createrepo", ".", cwd=create_repo_path) + + if nightly_build_from: + repo_domain = os.environ.get("SALT_REPO_DOMAIN_RELEASE", "repo.saltproject.io") + else: + repo_domain = os.environ.get( + "SALT_REPO_DOMAIN_STAGING", "staging.repo.saltproject.io" + ) + + salt_repo_user = os.environ.get("SALT_REPO_USER") + if salt_repo_user: + log.info( + "SALT_REPO_USER: %s", + salt_repo_user[0] + "*" * (len(salt_repo_user) - 2) + salt_repo_user[-1], + ) + salt_repo_pass = os.environ.get("SALT_REPO_PASS") + if salt_repo_pass: + log.info( + "SALT_REPO_PASS: %s", + salt_repo_pass[0] + "*" * (len(salt_repo_pass) - 2) + salt_repo_pass[-1], + ) + if salt_repo_user and salt_repo_pass: + repo_domain = f"{salt_repo_user}:{salt_repo_pass}@{repo_domain}" + + def _create_repo_file(create_repo_path, url_suffix): + ctx.info(f"Creating '{repo_file_path.relative_to(repo_path)}' file ...") + if nightly_build_from: + base_url = f"salt-dev/{nightly_build_from}/" + repo_file_contents = "[salt-nightly-repo]" + elif "rc" in salt_version: + base_url = "salt_rc/" + repo_file_contents = "[salt-rc-repo]" + else: + base_url = "" + repo_file_contents = "[salt-repo]" + base_url += f"salt/py3/{distro}/{distro_version}/{distro_arch}/{url_suffix}" + if distro == "amazon": + distro_name = "Amazon Linux" + elif distro == "redhat": + distro_name = "RHEL/CentOS" + else: + distro_name = distro.capitalize() + + if distro != "photon" and int(distro_version) < 8: + failovermethod = "\n failovermethod=priority" + else: + failovermethod = "" + + repo_file_contents += textwrap.dedent( + f""" + name=Salt repo for {distro_name} {distro_version} PY3 + baseurl=https://{repo_domain}/{base_url} + skip_if_unavailable=True{failovermethod} + priority=10 + enabled=1 + enabled_metadata=1 + gpgcheck=1 + gpgkey=https://{repo_domain}/{base_url}/{tools.utils.GPG_KEY_FILENAME}.pub + """ + ) + create_repo_path.write_text(repo_file_contents) + + if nightly_build_from: + repo_file_path = create_repo_path.parent / "nightly.repo" + else: + repo_file_path = create_repo_path.parent / f"{create_repo_path.name}.repo" + + _create_repo_file(repo_file_path, f"minor/{salt_version}") + + if not nightly_build_from: + remote_versions = _get_remote_versions( + tools.utils.STAGING_BUCKET_NAME, + create_repo_path.parent.relative_to(repo_path), + ) + major_version = Version(salt_version).major + matching_major = None + for version in remote_versions: + if version.major == major_version: + matching_major = version + break + if not matching_major or matching_major <= salt_version: + major_link = create_repo_path.parent.parent / str(major_version) + ctx.info(f"Creating '{major_link.relative_to(repo_path)}' symlink ...") + major_link.symlink_to(f"minor/{salt_version}") + repo_file_path = create_repo_path.parent.parent / f"{major_version}.repo" + _create_repo_file(repo_file_path, str(major_version)) + if not remote_versions or remote_versions[0] <= salt_version: + latest_link = create_repo_path.parent.parent / "latest" + ctx.info(f"Creating '{latest_link.relative_to(repo_path)}' symlink ...") + latest_link.symlink_to(f"minor/{salt_version}") + repo_file_path = create_repo_path.parent.parent / "latest.repo" + _create_repo_file(repo_file_path, "latest") + + ctx.info("Done") + + +@create.command( + name="windows", + arguments={ + "salt_version": { + "help": "The salt version for which to build the repository", + "required": True, + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + "nightly_build_from": { + "help": "Developement repository target", + }, + }, +) +def windows( + ctx: Context, + salt_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + nightly_build_from: str = None, +): + """ + Create the windows repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + _create_onedir_based_repo( + ctx, + salt_version=salt_version, + nightly_build_from=nightly_build_from, + repo_path=repo_path, + incoming=incoming, + key_id=key_id, + distro="windows", + pkg_suffixes=(".msi", ".exe"), + ) + ctx.info("Done") + + +@create.command( + name="macos", + arguments={ + "salt_version": { + "help": "The salt version for which to build the repository", + "required": True, + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + "nightly_build_from": { + "help": "Developement repository target", + }, + }, +) +def macos( + ctx: Context, + salt_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + nightly_build_from: str = None, +): + """ + Create the windows repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + _create_onedir_based_repo( + ctx, + salt_version=salt_version, + nightly_build_from=nightly_build_from, + repo_path=repo_path, + incoming=incoming, + key_id=key_id, + distro="macos", + pkg_suffixes=(".pkg",), + ) + ctx.info("Done") + + +@create.command( + name="onedir", + arguments={ + "salt_version": { + "help": "The salt version for which to build the repository", + "required": True, + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + "nightly_build_from": { + "help": "Developement repository target", + }, + }, +) +def onedir( + ctx: Context, + salt_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + nightly_build_from: str = None, +): + """ + Create the onedir repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + _create_onedir_based_repo( + ctx, + salt_version=salt_version, + nightly_build_from=nightly_build_from, + repo_path=repo_path, + incoming=incoming, + key_id=key_id, + distro="onedir", + pkg_suffixes=(".xz", ".zip"), + ) + ctx.info("Done") + + +@create.command( + name="src", + arguments={ + "salt_version": { + "help": "The salt version for which to build the repository", + "required": True, + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + "nightly_build_from": { + "help": "Developement repository target", + }, + }, +) +def src( + ctx: Context, + salt_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + nightly_build_from: str = None, +): + """ + Create the onedir repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + + ctx.info("Creating repository directory structure ...") + create_repo_path = create_top_level_repo_path( + ctx, + repo_path, + salt_version, + distro="src", + nightly_build_from=nightly_build_from, + ) + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + create_repo_path = create_repo_path / salt_version + create_repo_path.mkdir(exist_ok=True, parents=True) + hashes_base_path = create_repo_path / f"salt-{salt_version}" + for fpath in incoming.iterdir(): + if fpath.suffix not in (".gz",): + continue + ctx.info(f"* Processing {fpath} ...") + dpath = create_repo_path / fpath.name + ctx.info(f"Copying {fpath} to {dpath} ...") + shutil.copyfile(fpath, dpath) + for hash_name in ("blake2b", "sha512", "sha3_512"): + ctx.info(f" * Calculating {hash_name} ...") + hexdigest = _get_file_checksum(fpath, hash_name) + with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh: + wfh.write(f"{hexdigest} {dpath.name}\n") + with open(f"{dpath}.{hash_name}", "a+") as wfh: + wfh.write(f"{hexdigest} {dpath.name}\n") + + for fpath in create_repo_path.iterdir(): + if fpath.suffix in (".pub", ".gpg"): + continue + tools.utils.gpg_sign(ctx, key_id, fpath) + + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + ctx.info("Done") + + +def _get_remote_versions(bucket_name: str, remote_path: str): + log.info( + "Getting remote versions from bucket %r under path: %s", + bucket_name, + remote_path, + ) + remote_path = str(remote_path) + if not remote_path.endswith("/"): + remote_path += "/" + + s3 = boto3.client("s3") + ret = s3.list_objects( + Bucket=bucket_name, + Delimiter="/", + Prefix=remote_path, + ) + if "CommonPrefixes" not in ret: + return [] + versions = [] + for entry in ret["CommonPrefixes"]: + _, version = entry["Prefix"].rstrip("/").rsplit("/", 1) + if version == "latest": + continue + versions.append(Version(version)) + versions.sort(reverse=True) + log.info("Remote versions collected: %s", versions) + return versions + + +def _create_onedir_based_repo( + ctx: Context, + salt_version: str, + nightly_build_from: str | None, + repo_path: pathlib.Path, + incoming: pathlib.Path, + key_id: str, + distro: str, + pkg_suffixes: tuple[str, ...], +): + ctx.info("Creating repository directory structure ...") + create_repo_path = create_top_level_repo_path( + ctx, + repo_path, + salt_version, + distro, + nightly_build_from=nightly_build_from, + ) + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + + create_repo_path = create_full_repo_path( + ctx, + repo_path, + salt_version, + distro, + nightly_build_from=nightly_build_from, + ) + if not nightly_build_from: + repo_json_path = create_repo_path.parent.parent / "repo.json" + else: + repo_json_path = create_repo_path.parent / "repo.json" + + if nightly_build_from: + bucket_name = tools.utils.RELEASE_BUCKET_NAME + else: + bucket_name = tools.utils.STAGING_BUCKET_NAME + + release_json = {} + + copy_exclusions = ( + ".blake2b", + ".sha512", + ".sha3_512", + ".BLAKE2B", + ".SHA512", + ".SHA3_512", + ".json", + ) + hashes_base_path = create_repo_path / f"salt-{salt_version}" + for fpath in incoming.iterdir(): + if fpath.suffix in copy_exclusions: + continue + ctx.info(f"* Processing {fpath} ...") + dpath = create_repo_path / fpath.name + ctx.info(f"Copying {fpath} to {dpath} ...") + shutil.copyfile(fpath, dpath) + if "-amd64" in dpath.name.lower(): + arch = "amd64" + elif "-x86_64" in dpath.name.lower(): + arch = "x86_64" + elif "-x86" in dpath.name.lower(): + arch = "x86" + elif "-aarch64" in dpath.name.lower(): + arch = "aarch64" + else: + ctx.error( + f"Cannot pickup the right architecture from the filename '{dpath.name}'." + ) + ctx.exit(1) + if distro == "onedir": + if "-onedir-linux-" in dpath.name.lower(): + release_os = "linux" + elif "-onedir-darwin-" in dpath.name.lower(): + release_os = "macos" + elif "-onedir-windows-" in dpath.name.lower(): + release_os = "windows" + else: + ctx.error( + f"Cannot pickup the right OS from the filename '{dpath.name}'." + ) + ctx.exit(1) + else: + release_os = distro + release_json[dpath.name] = { + "name": dpath.name, + "version": salt_version, + "os": release_os, + "arch": arch, + } + for hash_name in ("blake2b", "sha512", "sha3_512"): + ctx.info(f" * Calculating {hash_name} ...") + hexdigest = _get_file_checksum(fpath, hash_name) + release_json[dpath.name][hash_name.upper()] = hexdigest + with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh: + wfh.write(f"{hexdigest} {dpath.name}\n") + with open(f"{dpath}.{hash_name}", "a+") as wfh: + wfh.write(f"{hexdigest} {dpath.name}\n") + + for fpath in create_repo_path.iterdir(): + if fpath.suffix in pkg_suffixes: + continue + tools.utils.gpg_sign(ctx, key_id, fpath) + + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, create_repo_path) + + repo_json = get_repo_json_file_contents( + ctx, bucket_name=bucket_name, repo_path=repo_path, repo_json_path=repo_json_path + ) + if nightly_build_from: + ctx.info(f"Writing {repo_json_path} ...") + repo_json_path.write_text(json.dumps(repo_json, sort_keys=True)) + return + + major_version = Version(salt_version).major + minor_repo_json_path = create_repo_path.parent / "repo.json" + minor_repo_json = get_repo_json_file_contents( + ctx, + bucket_name=bucket_name, + repo_path=repo_path, + repo_json_path=minor_repo_json_path, + ) + minor_repo_json[salt_version] = release_json + versions = parse_versions(*list(minor_repo_json)) + ctx.info( + f"Collected versions from {minor_repo_json_path.relative_to(repo_path)}: " + f"{', '.join(str(vs) for vs in versions)}" + ) + minor_versions = [v for v in versions if v.major == major_version] + ctx.info( + f"Collected versions(Matching major: {major_version}) from " + f"{minor_repo_json_path.relative_to(repo_path)}: " + f"{', '.join(str(vs) for vs in minor_versions)}" + ) + if not versions: + latest_version = Version(salt_version) + else: + latest_version = versions[0] + if not minor_versions: + latest_minor_version = Version(salt_version) + else: + latest_minor_version = minor_versions[0] + + ctx.info(f"Release Version: {salt_version}") + ctx.info(f"Latest Repo Version: {latest_version}") + ctx.info(f"Latest Release Minor Version: {latest_minor_version}") + + latest_link = create_repo_path.parent.parent / "latest" + if latest_version <= salt_version: + repo_json["latest"] = release_json + ctx.info(f"Creating '{latest_link.relative_to(repo_path)}' symlink ...") + if latest_link.exists(): + latest_link.unlink() + latest_link.symlink_to(f"minor/{salt_version}") + else: + ctx.info( + f"Not creating the '{latest_link.relative_to(repo_path)}' symlink " + f"since {latest_version} > {salt_version}" + ) + + major_link = create_repo_path.parent.parent / str(major_version) + if latest_minor_version <= salt_version: + minor_repo_json["latest"] = release_json + # This is the latest minor, update the major in the top level repo.json + # to this version + repo_json[str(major_version)] = release_json + ctx.info(f"Creating '{major_link.relative_to(repo_path)}' symlink ...") + if major_link.exists(): + major_link.unlink() + major_link.symlink_to(f"minor/{salt_version}") + else: + ctx.info( + f"Not creating the '{major_link.relative_to(repo_path)}' symlink " + f"since {latest_minor_version} > {salt_version}" + ) + + ctx.info(f"Writing {minor_repo_json_path} ...") + minor_repo_json_path.write_text(json.dumps(minor_repo_json, sort_keys=True)) + + ctx.info(f"Writing {repo_json_path} ...") + repo_json_path.write_text(json.dumps(repo_json, sort_keys=True)) + + +def _get_file_checksum(fpath: pathlib.Path, hash_name: str) -> str: + + with fpath.open("rb") as rfh: + try: + digest = hashlib.file_digest(rfh, hash_name) # type: ignore[attr-defined] + except AttributeError: + # Python < 3.11 + buf = bytearray(2**18) # Reusable buffer to reduce allocations. + view = memoryview(buf) + digest = getattr(hashlib, hash_name)() + while True: + size = rfh.readinto(buf) + if size == 0: + break # EOF + digest.update(view[:size]) + hexdigest: str = digest.hexdigest() + return hexdigest diff --git a/tools/pkg/repo/publish.py b/tools/pkg/repo/publish.py new file mode 100644 index 00000000000..cc6a92235c4 --- /dev/null +++ b/tools/pkg/repo/publish.py @@ -0,0 +1,653 @@ +""" +These commands are used to build the pacakge repository files. +""" +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import fnmatch +import json +import logging +import os +import pathlib +import re +import sys +import tempfile +import textwrap +from typing import TYPE_CHECKING, Any + +import packaging.version +from ptscripts import Context, command_group + +import tools.pkg +import tools.utils +from tools.utils import ( + Version, + create_full_repo_path, + get_repo_json_file_contents, + get_salt_releases, + parse_versions, +) + +try: + import boto3 + from botocore.exceptions import ClientError +except ImportError: + print( + "\nPlease run 'python -m pip install -r " + "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info), + file=sys.stderr, + flush=True, + ) + raise + +log = logging.getLogger(__name__) + +publish = command_group( + name="publish", + help="Packaging Repository Publication Related Commands", + parent=["pkg", "repo"], +) + + +@publish.command( + arguments={ + "repo_path": { + "help": "Local path for the repository that shall be published.", + }, + "salt_version": { + "help": "The salt version for which to build the repository", + "required": True, + }, + } +) +def nightly(ctx: Context, repo_path: pathlib.Path, salt_version: str = None): + """ + Publish to the nightly bucket. + """ + if TYPE_CHECKING: + assert salt_version is not None + _publish_repo( + ctx, repo_path=repo_path, nightly_build=True, salt_version=salt_version + ) + + +@publish.command( + arguments={ + "repo_path": { + "help": "Local path for the repository that shall be published.", + }, + "salt_version": { + "help": "The salt version for which to build the repository", + "required": True, + }, + } +) +def staging(ctx: Context, repo_path: pathlib.Path, salt_version: str = None): + """ + Publish to the staging bucket. + """ + if TYPE_CHECKING: + assert salt_version is not None + _publish_repo(ctx, repo_path=repo_path, stage=True, salt_version=salt_version) + + +@publish.command( + arguments={ + "salt_version": { + "help": "The salt version to release.", + }, + } +) +def release(ctx: Context, salt_version: str): + """ + Publish to the release bucket. + """ + if "rc" in salt_version: + bucket_folder = "salt_rc/salt/py3" + else: + bucket_folder = "salt/py3" + + files_to_copy: list[str] + directories_to_delete: list[str] = [] + + ctx.info("Grabbing remote file listing of files to copy...") + s3 = boto3.client("s3") + repo_release_files_path = pathlib.Path( + f"release-artifacts/{salt_version}/.release-files.json" + ) + repo_release_symlinks_path = pathlib.Path( + f"release-artifacts/{salt_version}/.release-symlinks.json" + ) + with tempfile.TemporaryDirectory(prefix=f"{salt_version}_release_") as tsd: + local_release_files_path = pathlib.Path(tsd) / repo_release_files_path.name + try: + bucket_name = tools.utils.STAGING_BUCKET_NAME + with local_release_files_path.open("wb") as wfh: + ctx.info( + f"Downloading {repo_release_files_path} from bucket {bucket_name} ..." + ) + s3.download_fileobj( + Bucket=bucket_name, + Key=str(repo_release_files_path), + Fileobj=wfh, + ) + files_to_copy = json.loads(local_release_files_path.read_text()) + except ClientError as exc: + if "Error" not in exc.response: + log.exception(f"Error downloading {repo_release_files_path}: {exc}") + ctx.exit(1) + if exc.response["Error"]["Code"] == "404": + ctx.error(f"Could not find {repo_release_files_path} in bucket.") + ctx.exit(1) + if exc.response["Error"]["Code"] == "400": + ctx.error( + f"Could not download {repo_release_files_path} from bucket: {exc}" + ) + ctx.exit(1) + log.exception(f"Error downloading {repo_release_files_path}: {exc}") + ctx.exit(1) + local_release_symlinks_path = ( + pathlib.Path(tsd) / repo_release_symlinks_path.name + ) + try: + with local_release_symlinks_path.open("wb") as wfh: + ctx.info( + f"Downloading {repo_release_symlinks_path} from bucket {bucket_name} ..." + ) + s3.download_fileobj( + Bucket=bucket_name, + Key=str(repo_release_symlinks_path), + Fileobj=wfh, + ) + directories_to_delete = json.loads(local_release_symlinks_path.read_text()) + except ClientError as exc: + if "Error" not in exc.response: + log.exception(f"Error downloading {repo_release_symlinks_path}: {exc}") + ctx.exit(1) + if exc.response["Error"]["Code"] == "404": + ctx.error(f"Could not find {repo_release_symlinks_path} in bucket.") + ctx.exit(1) + if exc.response["Error"]["Code"] == "400": + ctx.error( + f"Could not download {repo_release_symlinks_path} from bucket: {exc}" + ) + ctx.exit(1) + log.exception(f"Error downloading {repo_release_symlinks_path}: {exc}") + ctx.exit(1) + + if directories_to_delete: + with tools.utils.create_progress_bar() as progress: + task = progress.add_task( + "Deleting directories to override.", + total=len(directories_to_delete), + ) + for directory in directories_to_delete: + try: + objects_to_delete: list[dict[str, str]] = [] + for path in _get_repo_file_list( + bucket_name=tools.utils.RELEASE_BUCKET_NAME, + bucket_folder=bucket_folder, + glob_match=f"{directory}/**", + ): + objects_to_delete.append({"Key": path}) + if objects_to_delete: + s3.delete_objects( + Bucket=tools.utils.RELEASE_BUCKET_NAME, + Delete={"Objects": objects_to_delete}, + ) + except ClientError: + log.exception("Failed to delete remote files") + finally: + progress.update(task, advance=1) + + already_copied_files: list[str] = [] + s3 = boto3.client("s3") + dot_repo_files = [] + with tools.utils.create_progress_bar() as progress: + task = progress.add_task( + "Copying files between buckets", total=len(files_to_copy) + ) + for fpath in files_to_copy: + if fpath in already_copied_files: + continue + if fpath.endswith(".repo"): + dot_repo_files.append(fpath) + ctx.info(f" * Copying {fpath}") + try: + s3.copy_object( + Bucket=tools.utils.RELEASE_BUCKET_NAME, + Key=fpath, + CopySource={ + "Bucket": tools.utils.STAGING_BUCKET_NAME, + "Key": fpath, + }, + MetadataDirective="COPY", + TaggingDirective="COPY", + ServerSideEncryption="AES256", + ) + already_copied_files.append(fpath) + except ClientError: + log.exception(f"Failed to copy {fpath}") + finally: + progress.update(task, advance=1) + + # Now let's get the onedir based repositories where we need to update several repo.json + major_version = packaging.version.parse(salt_version).major + with tempfile.TemporaryDirectory(prefix=f"{salt_version}_release_") as tsd: + repo_path = pathlib.Path(tsd) + for distro in ("windows", "macos", "onedir"): + + create_repo_path = create_full_repo_path( + ctx, + repo_path, + salt_version, + distro=distro, + ) + repo_json_path = create_repo_path.parent.parent / "repo.json" + + release_repo_json = get_repo_json_file_contents( + ctx, + bucket_name=tools.utils.RELEASE_BUCKET_NAME, + repo_path=repo_path, + repo_json_path=repo_json_path, + ) + minor_repo_json_path = create_repo_path.parent / "repo.json" + + staging_minor_repo_json = get_repo_json_file_contents( + ctx, + bucket_name=tools.utils.STAGING_BUCKET_NAME, + repo_path=repo_path, + repo_json_path=minor_repo_json_path, + ) + release_minor_repo_json = get_repo_json_file_contents( + ctx, + bucket_name=tools.utils.RELEASE_BUCKET_NAME, + repo_path=repo_path, + repo_json_path=minor_repo_json_path, + ) + + release_json = staging_minor_repo_json[salt_version] + + major_version = Version(salt_version).major + versions = parse_versions(*list(release_minor_repo_json)) + ctx.info( + f"Collected versions from {minor_repo_json_path.relative_to(repo_path)}: " + f"{', '.join(str(vs) for vs in versions)}" + ) + minor_versions = [v for v in versions if v.major == major_version] + ctx.info( + f"Collected versions(Matching major: {major_version}) from " + f"{minor_repo_json_path.relative_to(repo_path)}: " + f"{', '.join(str(vs) for vs in minor_versions)}" + ) + if not versions: + latest_version = Version(salt_version) + else: + latest_version = versions[0] + if not minor_versions: + latest_minor_version = Version(salt_version) + else: + latest_minor_version = minor_versions[0] + + ctx.info(f"Release Version: {salt_version}") + ctx.info(f"Latest Repo Version: {latest_version}") + ctx.info(f"Latest Release Minor Version: {latest_minor_version}") + + # Add the minor version + release_minor_repo_json[salt_version] = release_json + + if latest_version <= salt_version: + release_repo_json["latest"] = release_json + + if latest_minor_version <= salt_version: + release_minor_repo_json["latest"] = release_json + + ctx.info(f"Writing {minor_repo_json_path} ...") + minor_repo_json_path.write_text( + json.dumps(release_minor_repo_json, sort_keys=True) + ) + ctx.info(f"Writing {repo_json_path} ...") + repo_json_path.write_text(json.dumps(release_repo_json, sort_keys=True)) + + # And now, let's get the several rpm "*.repo" files to update the base + # domain from staging to release + release_domain = os.environ.get( + "SALT_REPO_DOMAIN_RELEASE", "repo.saltproject.io" + ) + for path in dot_repo_files: + repo_file_path = repo_path.joinpath(path) + repo_file_path.parent.mkdir(exist_ok=True, parents=True) + bucket_name = tools.utils.STAGING_BUCKET_NAME + try: + ret = s3.head_object(Bucket=bucket_name, Key=path) + ctx.info( + f"Downloading existing '{repo_file_path.relative_to(repo_path)}' " + f"file from bucket {bucket_name}" + ) + size = ret["ContentLength"] + with repo_file_path.open("wb") as wfh: + with tools.utils.create_progress_bar( + file_progress=True + ) as progress: + task = progress.add_task( + description="Downloading...", total=size + ) + s3.download_fileobj( + Bucket=bucket_name, + Key=path, + Fileobj=wfh, + Callback=tools.utils.UpdateProgress(progress, task), + ) + updated_contents = re.sub( + r"^(baseurl|gpgkey)=https://([^/]+)/(.*)$", + rf"\1=https://{release_domain}/\3", + repo_file_path.read_text(), + flags=re.MULTILINE, + ) + ctx.info(f"Updated '{repo_file_path.relative_to(repo_path)}:") + ctx.print(updated_contents) + repo_file_path.write_text(updated_contents) + except ClientError as exc: + if "Error" not in exc.response: + raise + if exc.response["Error"]["Code"] != "404": + raise + ctx.info(f"Could not find {repo_file_path} in bucket {bucket_name}") + + for dirpath, dirnames, filenames in os.walk(repo_path, followlinks=True): + for path in filenames: + upload_path = pathlib.Path(dirpath, path) + relpath = upload_path.relative_to(repo_path) + size = upload_path.stat().st_size + ctx.info(f" {relpath}") + with tools.utils.create_progress_bar(file_progress=True) as progress: + task = progress.add_task(description="Uploading...", total=size) + s3.upload_file( + str(upload_path), + tools.utils.RELEASE_BUCKET_NAME, + str(relpath), + Callback=tools.utils.UpdateProgress(progress, task), + ) + + +@publish.command( + arguments={ + "salt_version": { + "help": "The salt version to release.", + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "repository": { + "help": ( + "The full repository name, ie, 'saltstack/salt' on GitHub " + "to run the checks against." + ) + }, + } +) +def github( + ctx: Context, + salt_version: str, + key_id: str = None, + repository: str = "saltstack/salt", +): + """ + Publish the release on GitHub releases. + """ + if TYPE_CHECKING: + assert key_id is not None + + s3 = boto3.client("s3") + + # Let's download the release artifacts stored in staging + artifacts_path = pathlib.Path.cwd() / "release-artifacts" + artifacts_path.mkdir(exist_ok=True) + release_artifacts_listing: dict[pathlib.Path, int] = {} + continuation_token = None + while True: + kwargs: dict[str, str] = {} + if continuation_token: + kwargs["ContinuationToken"] = continuation_token + ret = s3.list_objects_v2( + Bucket=tools.utils.STAGING_BUCKET_NAME, + Prefix=f"release-artifacts/{salt_version}", + FetchOwner=False, + **kwargs, + ) + contents = ret.pop("Contents", None) + if contents is None: + break + for entry in contents: + entry_path = pathlib.Path(entry["Key"]) + if entry_path.name.startswith("."): + continue + release_artifacts_listing[entry_path] = entry["Size"] + if not ret["IsTruncated"]: + break + continuation_token = ret["NextContinuationToken"] + + for entry_path, size in release_artifacts_listing.items(): + ctx.info(f" * {entry_path.name}") + local_path = artifacts_path / entry_path.name + with local_path.open("wb") as wfh: + with tools.utils.create_progress_bar(file_progress=True) as progress: + task = progress.add_task(description="Downloading...", total=size) + s3.download_fileobj( + Bucket=tools.utils.STAGING_BUCKET_NAME, + Key=str(entry_path), + Fileobj=wfh, + Callback=tools.utils.UpdateProgress(progress, task), + ) + + for artifact in artifacts_path.iterdir(): + if artifact.suffix in (".patch", ".asc", ".gpg", ".pub"): + continue + tools.utils.gpg_sign(ctx, key_id, artifact) + + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, artifacts_path) + + release_message = f"""\ + # Welcome to Salt v{salt_version} + + | :exclamation: ATTENTION | + |:-------------------------------------------------------------------------------------------------------------------------| + | The archives generated by GitHub(`Source code(zip)`, `Source code(tar.gz)`) will not report Salt's version properly. | + | Please use the tarball generated by The Salt Project Team(`salt-{salt_version}.tar.gz`). + """ + release_message_path = artifacts_path / "gh-release-body.md" + release_message_path.write_text(textwrap.dedent(release_message).strip()) + + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output is None: + ctx.warn("The 'GITHUB_OUTPUT' variable is not set. Stop processing.") + ctx.exit(0) + + if TYPE_CHECKING: + assert github_output is not None + + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"release-messsage-file={release_message_path.resolve()}\n") + + releases = get_salt_releases(ctx, repository) + if Version(salt_version) >= releases[-1]: + make_latest = True + else: + make_latest = False + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"make-latest={json.dumps(make_latest)}\n") + + artifacts_to_upload = [] + for artifact in artifacts_path.iterdir(): + if artifact.suffix == ".patch": + continue + if artifact.name == release_message_path.name: + continue + artifacts_to_upload.append(str(artifact.resolve())) + + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"release-artifacts={','.join(artifacts_to_upload)}\n") + ctx.exit(0) + + +def _get_repo_detailed_file_list( + bucket_name: str, + bucket_folder: str = "", + glob_match: str = "**", +) -> list[dict[str, Any]]: + s3 = boto3.client("s3") + listing: list[dict[str, Any]] = [] + continuation_token = None + while True: + kwargs: dict[str, str] = {} + if continuation_token: + kwargs["ContinuationToken"] = continuation_token + ret = s3.list_objects_v2( + Bucket=bucket_name, + Prefix=bucket_folder, + FetchOwner=False, + **kwargs, + ) + contents = ret.pop("Contents", None) + if contents is None: + break + for entry in contents: + if fnmatch.fnmatch(entry["Key"], glob_match): + listing.append(entry) + if not ret["IsTruncated"]: + break + continuation_token = ret["NextContinuationToken"] + return listing + + +def _get_repo_file_list( + bucket_name: str, bucket_folder: str, glob_match: str +) -> list[str]: + return [ + entry["Key"] + for entry in _get_repo_detailed_file_list( + bucket_name, bucket_folder, glob_match=glob_match + ) + ] + + +def _publish_repo( + ctx: Context, + repo_path: pathlib.Path, + salt_version: str, + nightly_build: bool = False, + stage: bool = False, +): + """ + Publish packaging repositories. + """ + if nightly_build: + bucket_name = tools.utils.RELEASE_BUCKET_NAME + elif stage: + bucket_name = tools.utils.STAGING_BUCKET_NAME + else: + bucket_name = tools.utils.RELEASE_BUCKET_NAME + + ctx.info("Preparing upload ...") + s3 = boto3.client("s3") + to_delete_paths: dict[pathlib.Path, list[dict[str, str]]] = {} + to_upload_paths: list[pathlib.Path] = [] + symlink_paths: list[str] = [] + uploaded_files: list[str] = [] + for dirpath, dirnames, filenames in os.walk(repo_path, followlinks=True): + for dirname in dirnames: + path = pathlib.Path(dirpath, dirname) + if not path.is_symlink(): + continue + # This is a symlink, then we need to delete all files under + # that directory in S3 because S3 does not understand symlinks + # and we would end up adding files to that folder instead of + # replacing it. + try: + relpath = path.relative_to(repo_path) + ret = s3.list_objects( + Bucket=bucket_name, + Prefix=str(relpath), + ) + if "Contents" not in ret: + continue + objects = [] + for entry in ret["Contents"]: + objects.append({"Key": entry["Key"]}) + to_delete_paths[path] = objects + symlink_paths.append(str(relpath)) + except ClientError as exc: + if "Error" not in exc.response: + raise + if exc.response["Error"]["Code"] != "404": + raise + + for fpath in filenames: + path = pathlib.Path(dirpath, fpath) + to_upload_paths.append(path) + + with tools.utils.create_progress_bar() as progress: + task = progress.add_task( + "Deleting directories to override.", total=len(to_delete_paths) + ) + for base, objects in to_delete_paths.items(): + relpath = base.relative_to(repo_path) + bucket_uri = f"s3://{bucket_name}/{relpath}" + progress.update(task, description=f"Deleting {bucket_uri}") + try: + ret = s3.delete_objects( + Bucket=bucket_name, + Delete={"Objects": objects}, + ) + except ClientError: + log.exception(f"Failed to delete {bucket_uri}") + finally: + progress.update(task, advance=1) + + try: + ctx.info("Uploading repository ...") + for upload_path in to_upload_paths: + relpath = upload_path.relative_to(repo_path) + size = upload_path.stat().st_size + ctx.info(f" {relpath}") + with tools.utils.create_progress_bar(file_progress=True) as progress: + task = progress.add_task(description="Uploading...", total=size) + s3.upload_file( + str(upload_path), + bucket_name, + str(relpath), + Callback=tools.utils.UpdateProgress(progress, task), + ExtraArgs={ + "Metadata": { + "x-amz-meta-salt-release-version": salt_version, + } + }, + ) + uploaded_files.append(str(relpath)) + if stage is True: + repo_files_path = f"release-artifacts/{salt_version}/.release-files.json" + ctx.info(f"Uploading {repo_files_path} ...") + s3.put_object( + Key=repo_files_path, + Bucket=bucket_name, + Body=json.dumps(uploaded_files).encode(), + Metadata={ + "x-amz-meta-salt-release-version": salt_version, + }, + ) + repo_symlinks_path = ( + f"release-artifacts/{salt_version}/.release-symlinks.json" + ) + ctx.info(f"Uploading {repo_symlinks_path} ...") + s3.put_object( + Key=repo_symlinks_path, + Bucket=bucket_name, + Body=json.dumps(symlink_paths).encode(), + Metadata={ + "x-amz-meta-salt-release-version": salt_version, + }, + ) + except KeyboardInterrupt: + pass diff --git a/tools/utils.py b/tools/utils.py index cb4379c61e0..28a79745844 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -1,8 +1,12 @@ # pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated from __future__ import annotations +import json import os import pathlib +import sys +from datetime import datetime +from typing import Any import packaging.version from ptscripts import Context @@ -16,6 +20,18 @@ from rich.progress import ( TransferSpeedColumn, ) +try: + import boto3 + from botocore.exceptions import ClientError +except ImportError: + print( + "\nPlease run 'python -m pip install -r " + "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info), + file=sys.stderr, + flush=True, + ) + raise + REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent GPG_KEY_FILENAME = "SALT-PROJECT-GPG-PUBKEY-2023" SPB_ENVIRONMENT = os.environ.get("SPB_ENVIRONMENT") or "prod" @@ -169,3 +185,114 @@ def get_salt_releases(ctx: Context, repository: str) -> list[Version]: # We're not going to parse dash or docs releases versions.add(Version(name)) return sorted(versions) + + +def parse_versions(*versions: str) -> list[Version]: + _versions = [] + for version in set(versions): + if version == "latest": + continue + _versions.append(Version(version)) + if _versions: + _versions.sort(reverse=True) + return _versions + + +def get_repo_json_file_contents( + ctx: Context, + bucket_name: str, + repo_path: pathlib.Path, + repo_json_path: pathlib.Path, +) -> dict[str, Any]: + s3 = boto3.client("s3") + repo_json: dict[str, Any] = {} + try: + ret = s3.head_object( + Bucket=bucket_name, Key=str(repo_json_path.relative_to(repo_path)) + ) + ctx.info( + f"Downloading existing '{repo_json_path.relative_to(repo_path)}' file " + f"from bucket {bucket_name}" + ) + size = ret["ContentLength"] + with repo_json_path.open("wb") as wfh: + with create_progress_bar(file_progress=True) as progress: + task = progress.add_task(description="Downloading...", total=size) + s3.download_fileobj( + Bucket=bucket_name, + Key=str(repo_json_path.relative_to(repo_path)), + Fileobj=wfh, + Callback=UpdateProgress(progress, task), + ) + with repo_json_path.open() as rfh: + repo_json = json.load(rfh) + except ClientError as exc: + if "Error" not in exc.response: + raise + if exc.response["Error"]["Code"] != "404": + raise + ctx.info(f"Could not find {repo_json_path} in bucket {bucket_name}") + if repo_json: + ctx.print(repo_json, soft_wrap=True) + return repo_json + + +def create_top_level_repo_path( + ctx: Context, + repo_path: pathlib.Path, + salt_version: str, + distro: str, + distro_version: str | None = None, # pylint: disable=bad-whitespace + distro_arch: str | None = None, # pylint: disable=bad-whitespace + nightly_build_from: str | None = None, # pylint: disable=bad-whitespace +): + create_repo_path = repo_path + if nightly_build_from: + create_repo_path = ( + create_repo_path + / "salt-dev" + / nightly_build_from + / datetime.utcnow().strftime("%Y-%m-%d") + ) + create_repo_path.mkdir(exist_ok=True, parents=True) + with ctx.chdir(create_repo_path.parent): + latest_nightly_symlink = pathlib.Path("latest") + if not latest_nightly_symlink.exists(): + ctx.info( + f"Creating 'latest' symlink to '{create_repo_path.relative_to(repo_path)}' ..." + ) + latest_nightly_symlink.symlink_to( + create_repo_path.name, target_is_directory=True + ) + elif "rc" in salt_version: + create_repo_path = create_repo_path / "salt_rc" + create_repo_path = create_repo_path / "salt" / "py3" / distro + if distro_version: + create_repo_path = create_repo_path / distro_version + if distro_arch: + create_repo_path = create_repo_path / distro_arch + create_repo_path.mkdir(exist_ok=True, parents=True) + return create_repo_path + + +def create_full_repo_path( + ctx: Context, + repo_path: pathlib.Path, + salt_version: str, + distro: str, + distro_version: str | None = None, # pylint: disable=bad-whitespace + distro_arch: str | None = None, # pylint: disable=bad-whitespace + nightly_build_from: str | None = None, # pylint: disable=bad-whitespace +): + create_repo_path = create_top_level_repo_path( + ctx, + repo_path, + salt_version, + distro, + distro_version, + distro_arch, + nightly_build_from=nightly_build_from, + ) + create_repo_path = create_repo_path / "minor" / salt_version + create_repo_path.mkdir(exist_ok=True, parents=True) + return create_repo_path From 4896c90684a955111ef7a8cdcd9da9ca5d475c99 Mon Sep 17 00:00:00 2001 From: MKLeb Date: Mon, 8 May 2023 15:30:25 -0400 Subject: [PATCH 026/152] Address review comments (typos, docs) --- tools/pkg/repo/__init__.py | 2 +- tools/pkg/repo/create.py | 2 +- tools/pkg/repo/publish.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/pkg/repo/__init__.py b/tools/pkg/repo/__init__.py index 8a3cbd9c81f..d965fcfd923 100644 --- a/tools/pkg/repo/__init__.py +++ b/tools/pkg/repo/__init__.py @@ -1,5 +1,5 @@ """ -These commands are used to build the pacakge repository files. +These commands are used to build the package repository files. """ # pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated from __future__ import annotations diff --git a/tools/pkg/repo/create.py b/tools/pkg/repo/create.py index ec4b3331c42..60ed8ad0570 100644 --- a/tools/pkg/repo/create.py +++ b/tools/pkg/repo/create.py @@ -1,5 +1,5 @@ """ -These commands are used to build the pacakge repository files. +These commands are used to build the package repository files. """ # pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated from __future__ import annotations diff --git a/tools/pkg/repo/publish.py b/tools/pkg/repo/publish.py index cc6a92235c4..1c87d20b490 100644 --- a/tools/pkg/repo/publish.py +++ b/tools/pkg/repo/publish.py @@ -1,5 +1,5 @@ """ -These commands are used to build the pacakge repository files. +These commands are used to build the package repository files. """ # pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated from __future__ import annotations @@ -55,7 +55,7 @@ publish = command_group( "help": "Local path for the repository that shall be published.", }, "salt_version": { - "help": "The salt version for which to build the repository", + "help": "The salt version of the repository to publish", "required": True, }, } @@ -77,7 +77,7 @@ def nightly(ctx: Context, repo_path: pathlib.Path, salt_version: str = None): "help": "Local path for the repository that shall be published.", }, "salt_version": { - "help": "The salt version for which to build the repository", + "help": "The salt version of the repository to publish", "required": True, }, } From 33e2538aa873c6269777ddd05bb7293ff7bbcc87 Mon Sep 17 00:00:00 2001 From: Frode Gundersen Date: Thu, 23 Feb 2023 22:49:52 +0000 Subject: [PATCH 027/152] migrate unit_states_test_linux_acl to pytest --- tests/pytests/unit/states/test_linux_acl.py | 539 ++++++++++++++++++ tests/unit/states/test_linux_acl.py | 589 -------------------- 2 files changed, 539 insertions(+), 589 deletions(-) create mode 100644 tests/pytests/unit/states/test_linux_acl.py delete mode 100644 tests/unit/states/test_linux_acl.py diff --git a/tests/pytests/unit/states/test_linux_acl.py b/tests/pytests/unit/states/test_linux_acl.py new file mode 100644 index 00000000000..976a57b8c4b --- /dev/null +++ b/tests/pytests/unit/states/test_linux_acl.py @@ -0,0 +1,539 @@ +""" + :codeauthor: Jayesh Kariya + + Test cases for salt.states.linux_acl +""" + +import pytest + +import salt.states.linux_acl as linux_acl +from salt.exceptions import CommandExecutionError +from tests.support.mock import MagicMock, patch + +pytestmark = [ + pytest.mark.skip_unless_on_linux( + reason="Only run on Linux", + ) +] + + +@pytest.fixture +def configure_loader_modules(): + return {linux_acl: {}} + + +def test_present(): + """ + Test to ensure a Linux ACL is present + """ + maxDiff = None + name = "/root" + acl_type = "users" + acl_name = "damian" + perms = "rwx" + + mock = MagicMock( + side_effect=[ + {name: {acl_type: [{acl_name: {"octal": 5}}]}}, + {name: {acl_type: [{acl_name: {"octal": 5}}]}}, + {name: {acl_type: [{acl_name: {"octal": 5}}]}}, + {name: {acl_type: [{}]}}, + {name: {acl_type: [{}]}}, + {name: {acl_type: [{}]}}, + { + name: {acl_type: [{acl_name: {"octal": 7}}]}, + name + "/foo": {acl_type: [{acl_name: {"octal": 5}}]}, + }, + { + name: {acl_type: [{acl_name: {"octal": 7}}]}, + name + "/foo": {acl_type: [{acl_name: {"octal": 7}}]}, + }, + {name: {acl_type: ""}}, + { + name: {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, + name + "/foo": {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, + }, + { + name: {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, + name + "/foo": {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, + }, + { + name: {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, + name + "/foo": {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, + }, + ] + ) + mock_modfacl = MagicMock(return_value=True) + + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Updated permissions will be applied for {}: r-x -> {}".format( + acl_name, perms + ) + ret = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": perms, + }, + "old": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": "r-x", + }, + }, + "result": None, + } + + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + # Update - test=False + with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Updated permissions for {}".format(acl_name) + ret = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": perms, + }, + "old": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": "r-x", + }, + }, + "result": True, + } + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + # Update - modfacl error + with patch.dict( + linux_acl.__salt__, + {"acl.modfacl": MagicMock(side_effect=CommandExecutionError("Custom err"))}, + ): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Error updating permissions for {}: Custom err".format(acl_name) + ret = { + "name": name, + "comment": comt, + "changes": {}, + "result": False, + } + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + # New - test=True + with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "New permissions will be applied for {}: {}".format( + acl_name, perms + ) + ret = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": perms, + } + }, + "result": None, + } + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + # New - test=False + with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Applied new permissions for {}".format(acl_name) + ret = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": perms, + } + }, + "result": True, + } + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + # New - modfacl error + with patch.dict( + linux_acl.__salt__, + {"acl.modfacl": MagicMock(side_effect=CommandExecutionError("Custom err"))}, + ): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Error updating permissions for {}: Custom err".format(acl_name) + ret = { + "name": name, + "comment": comt, + "changes": {}, + "result": False, + } + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + + # New - recurse true + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Updated permissions will be applied for {}: rwx -> {}".format( + acl_name, perms + ) + ret = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": perms, + }, + "old": { + "acl_name": acl_name, + "acl_type": acl_type, + "perms": "rwx", + }, + }, + "result": None, + } + + assert ( + linux_acl.present(name, acl_type, acl_name, perms, recurse=True) + == ret + ) + + # New - recurse true - nothing to do + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Permissions are in the desired state" + ret = {"name": name, "comment": comt, "changes": {}, "result": True} + + assert ( + linux_acl.present(name, acl_type, acl_name, perms, recurse=True) + == ret + ) + + # No acl type + comt = "ACL Type does not exist" + ret = {"name": name, "comment": comt, "result": False, "changes": {}} + assert linux_acl.present(name, acl_type, acl_name, perms) == ret + + # default recurse false - nothing to do + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Permissions are in the desired state" + ret = {"name": name, "comment": comt, "changes": {}, "result": True} + + assert ( + linux_acl.present( + name, "d:" + acl_type, acl_name, perms, recurse=False + ) + == ret + ) + + # default recurse false - nothing to do + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Permissions are in the desired state" + ret = {"name": name, "comment": comt, "changes": {}, "result": True} + + assert ( + linux_acl.present( + name, "d:" + acl_type, acl_name, perms, recurse=False + ) + == ret + ) + + # default recurse true - nothing to do + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Permissions are in the desired state" + ret = {"name": name, "comment": comt, "changes": {}, "result": True} + + assert ( + linux_acl.present( + name, "d:" + acl_type, acl_name, perms, recurse=True + ) + == ret + ) + + +def test_absent(): + """ + Test to ensure a Linux ACL does not exist + """ + name = "/root" + acl_type = "users" + acl_name = "damian" + perms = "rwx" + + ret = {"name": name, "result": None, "comment": "", "changes": {}} + + mock = MagicMock( + side_effect=[ + {name: {acl_type: [{acl_name: {"octal": "A"}}]}}, + {name: {acl_type: ""}}, + ] + ) + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Removing permissions" + ret.update({"comment": comt}) + assert linux_acl.absent(name, acl_type, acl_name, perms) == ret + + comt = "ACL Type does not exist" + ret.update({"comment": comt, "result": False}) + assert linux_acl.absent(name, acl_type, acl_name, perms) == ret + + +def test_list_present(): + """ + Test to ensure a Linux ACL is present + """ + maxDiff = None + name = "/root" + acl_type = "user" + acl_names = ["root", "damian", "homer"] + acl_comment = {"owner": "root", "group": "root", "file": "/root"} + perms = "rwx" + + mock = MagicMock( + side_effect=[ + { + name: { + acl_type: [ + {acl_names[0]: {"octal": "A"}}, + {acl_names[1]: {"octal": "A"}}, + {acl_names[2]: {"octal": "A"}}, + ], + "comment": acl_comment, + } + }, + { + name: { + acl_type: [ + {acl_names[0]: {"octal": "A"}}, + {acl_names[1]: {"octal": "A"}}, + ], + "comment": acl_comment, + } + }, + { + name: { + acl_type: [ + {acl_names[0]: {"octal": "A"}}, + {acl_names[1]: {"octal": "A"}}, + ] + } + }, + {name: {acl_type: [{}]}}, + {name: {acl_type: [{}]}}, + {name: {acl_type: [{}]}}, + {name: {acl_type: ""}}, + ] + ) + mock_modfacl = MagicMock(return_value=True) + + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + # Update - test=True + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Updated permissions will be applied for {}: A -> {}".format( + acl_names, perms + ) + expected = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": ", ".join(acl_names), + "acl_type": acl_type, + "perms": 7, + }, + "old": { + "acl_name": ", ".join(acl_names), + "acl_type": acl_type, + "perms": "A", + }, + }, + "result": None, + } + + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert ret == expected + + # Update - test=False + with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Applied new permissions for {}".format(", ".join(acl_names)) + expected = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": ", ".join(acl_names), + "acl_type": acl_type, + "perms": "rwx", + } + }, + "result": True, + } + + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert expected == ret + + # Update - modfacl error + with patch.dict( + linux_acl.__salt__, + {"acl.modfacl": MagicMock(side_effect=CommandExecutionError("Custom err"))}, + ): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Error updating permissions for {}: Custom err".format(acl_names) + expected = { + "name": name, + "comment": comt, + "changes": {}, + "result": False, + } + + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert expected == ret + + # New - test=True + with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "New permissions will be applied for {}: {}".format( + acl_names, perms + ) + expected = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": ", ".join(acl_names), + "acl_type": acl_type, + "perms": perms, + } + }, + "result": None, + } + + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert expected == ret + + # New - test=False + with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Applied new permissions for {}".format(", ".join(acl_names)) + expected = { + "name": name, + "comment": comt, + "changes": { + "new": { + "acl_name": ", ".join(acl_names), + "acl_type": acl_type, + "perms": perms, + } + }, + "result": True, + } + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert expected == ret + + # New - modfacl error + with patch.dict( + linux_acl.__salt__, + {"acl.modfacl": MagicMock(side_effect=CommandExecutionError("Custom err"))}, + ): + with patch.dict(linux_acl.__opts__, {"test": False}): + comt = "Error updating permissions for {}: Custom err".format(acl_names) + expected = { + "name": name, + "comment": comt, + "changes": {}, + "result": False, + } + + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert expected == ret + + # No acl type + comt = "ACL Type does not exist" + expected = { + "name": name, + "comment": comt, + "result": False, + "changes": {}, + } + ret = linux_acl.list_present(name, acl_type, acl_names, perms) + assert expected == ret + + +def test_list_absent(): + """ + Test to ensure a Linux ACL does not exist + """ + name = "/root" + acl_type = "users" + acl_names = ["damian", "homer"] + perms = "rwx" + + ret = {"name": name, "result": None, "comment": "", "changes": {}} + + mock = MagicMock( + side_effect=[ + { + name: { + acl_type: [ + {acl_names[0]: {"octal": "A"}, acl_names[1]: {"octal": "A"}} + ] + } + }, + {name: {acl_type: ""}}, + ] + ) + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Removing permissions" + ret.update({"comment": comt}) + assert linux_acl.list_absent(name, acl_type, acl_names, perms) == ret + + comt = "ACL Type does not exist" + ret.update({"comment": comt, "result": False}) + assert linux_acl.list_absent(name, acl_type, acl_names) == ret + + +def test_absent_recursive(): + """ + Test to ensure a Linux ACL does not exist + """ + name = "/root" + acl_type = "users" + acl_name = "damian" + perms = "rwx" + + ret = {"name": name, "result": None, "comment": "", "changes": {}} + + mock = MagicMock( + side_effect=[ + { + name: {acl_type: [{acl_name: {"octal": 7}}]}, + name + "/foo": {acl_type: [{acl_name: {"octal": "A"}}]}, + } + ] + ) + with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): + with patch.dict(linux_acl.__opts__, {"test": True}): + comt = "Removing permissions" + ret.update({"comment": comt}) + assert ( + linux_acl.absent(name, acl_type, acl_name, perms, recurse=True) == ret + ) diff --git a/tests/unit/states/test_linux_acl.py b/tests/unit/states/test_linux_acl.py deleted file mode 100644 index 2961fbad53a..00000000000 --- a/tests/unit/states/test_linux_acl.py +++ /dev/null @@ -1,589 +0,0 @@ -""" - :codeauthor: Jayesh Kariya -""" - -import pytest - -import salt.states.linux_acl as linux_acl -from salt.exceptions import CommandExecutionError -from tests.support.mixins import LoaderModuleMockMixin -from tests.support.mock import MagicMock, patch -from tests.support.unit import TestCase - - -@pytest.mark.skip_unless_on_linux -class LinuxAclTestCase(TestCase, LoaderModuleMockMixin): - """ - Test cases for salt.states.linux_acl - """ - - def setup_loader_modules(self): - return {linux_acl: {}} - - # 'present' function tests: 1 - - def test_present(self): - """ - Test to ensure a Linux ACL is present - """ - self.maxDiff = None - name = "/root" - acl_type = "users" - acl_name = "damian" - perms = "rwx" - - mock = MagicMock( - side_effect=[ - {name: {acl_type: [{acl_name: {"octal": 5}}]}}, - {name: {acl_type: [{acl_name: {"octal": 5}}]}}, - {name: {acl_type: [{acl_name: {"octal": 5}}]}}, - {name: {acl_type: [{}]}}, - {name: {acl_type: [{}]}}, - {name: {acl_type: [{}]}}, - { - name: {acl_type: [{acl_name: {"octal": 7}}]}, - name + "/foo": {acl_type: [{acl_name: {"octal": 5}}]}, - }, - { - name: {acl_type: [{acl_name: {"octal": 7}}]}, - name + "/foo": {acl_type: [{acl_name: {"octal": 7}}]}, - }, - {name: {acl_type: ""}}, - { - name: {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, - name + "/foo": {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, - }, - { - name: {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, - name + "/foo": {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, - }, - { - name: {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, - name + "/foo": {"defaults": {"users": [{acl_name: {"octal": 7}}]}}, - }, - ] - ) - mock_modfacl = MagicMock(return_value=True) - - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Updated permissions will be applied for {}: r-x -> {}".format( - acl_name, perms - ) - ret = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": perms, - }, - "old": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": "r-x", - }, - }, - "result": None, - } - - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - # Update - test=False - with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Updated permissions for {}".format(acl_name) - ret = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": perms, - }, - "old": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": "r-x", - }, - }, - "result": True, - } - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - # Update - modfacl error - with patch.dict( - linux_acl.__salt__, - { - "acl.modfacl": MagicMock( - side_effect=CommandExecutionError("Custom err") - ) - }, - ): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Error updating permissions for {}: Custom err".format( - acl_name - ) - ret = { - "name": name, - "comment": comt, - "changes": {}, - "result": False, - } - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - # New - test=True - with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "New permissions will be applied for {}: {}".format( - acl_name, perms - ) - ret = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": perms, - } - }, - "result": None, - } - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - # New - test=False - with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Applied new permissions for {}".format(acl_name) - ret = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": perms, - } - }, - "result": True, - } - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - # New - modfacl error - with patch.dict( - linux_acl.__salt__, - { - "acl.modfacl": MagicMock( - side_effect=CommandExecutionError("Custom err") - ) - }, - ): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Error updating permissions for {}: Custom err".format( - acl_name - ) - ret = { - "name": name, - "comment": comt, - "changes": {}, - "result": False, - } - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - - # New - recurse true - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = ( - "Updated permissions will be applied for {}: rwx -> {}".format( - acl_name, perms - ) - ) - ret = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": perms, - }, - "old": { - "acl_name": acl_name, - "acl_type": acl_type, - "perms": "rwx", - }, - }, - "result": None, - } - - self.assertDictEqual( - linux_acl.present( - name, acl_type, acl_name, perms, recurse=True - ), - ret, - ) - - # New - recurse true - nothing to do - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Permissions are in the desired state" - ret = {"name": name, "comment": comt, "changes": {}, "result": True} - - self.assertDictEqual( - linux_acl.present( - name, acl_type, acl_name, perms, recurse=True - ), - ret, - ) - - # No acl type - comt = "ACL Type does not exist" - ret = {"name": name, "comment": comt, "result": False, "changes": {}} - self.assertDictEqual( - linux_acl.present(name, acl_type, acl_name, perms), ret - ) - - # default recurse false - nothing to do - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Permissions are in the desired state" - ret = {"name": name, "comment": comt, "changes": {}, "result": True} - - self.assertDictEqual( - linux_acl.present( - name, "d:" + acl_type, acl_name, perms, recurse=False - ), - ret, - ) - - # default recurse false - nothing to do - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Permissions are in the desired state" - ret = {"name": name, "comment": comt, "changes": {}, "result": True} - - self.assertDictEqual( - linux_acl.present( - name, "d:" + acl_type, acl_name, perms, recurse=False - ), - ret, - ) - - # default recurse true - nothing to do - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Permissions are in the desired state" - ret = {"name": name, "comment": comt, "changes": {}, "result": True} - - self.assertDictEqual( - linux_acl.present( - name, "d:" + acl_type, acl_name, perms, recurse=True - ), - ret, - ) - - # 'absent' function tests: 2 - - def test_absent(self): - """ - Test to ensure a Linux ACL does not exist - """ - name = "/root" - acl_type = "users" - acl_name = "damian" - perms = "rwx" - - ret = {"name": name, "result": None, "comment": "", "changes": {}} - - mock = MagicMock( - side_effect=[ - {name: {acl_type: [{acl_name: {"octal": "A"}}]}}, - {name: {acl_type: ""}}, - ] - ) - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Removing permissions" - ret.update({"comment": comt}) - self.assertDictEqual( - linux_acl.absent(name, acl_type, acl_name, perms), ret - ) - - comt = "ACL Type does not exist" - ret.update({"comment": comt, "result": False}) - self.assertDictEqual(linux_acl.absent(name, acl_type, acl_name, perms), ret) - - # 'list_present' function tests: 1 - - def test_list_present(self): - """ - Test to ensure a Linux ACL is present - """ - self.maxDiff = None - name = "/root" - acl_type = "user" - acl_names = ["root", "damian", "homer"] - acl_comment = {"owner": "root", "group": "root", "file": "/root"} - perms = "rwx" - - mock = MagicMock( - side_effect=[ - { - name: { - acl_type: [ - {acl_names[0]: {"octal": "A"}}, - {acl_names[1]: {"octal": "A"}}, - {acl_names[2]: {"octal": "A"}}, - ], - "comment": acl_comment, - } - }, - { - name: { - acl_type: [ - {acl_names[0]: {"octal": "A"}}, - {acl_names[1]: {"octal": "A"}}, - ], - "comment": acl_comment, - } - }, - { - name: { - acl_type: [ - {acl_names[0]: {"octal": "A"}}, - {acl_names[1]: {"octal": "A"}}, - ] - } - }, - {name: {acl_type: [{}]}}, - {name: {acl_type: [{}]}}, - {name: {acl_type: [{}]}}, - {name: {acl_type: ""}}, - ] - ) - mock_modfacl = MagicMock(return_value=True) - - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - # Update - test=True - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Updated permissions will be applied for {}: A -> {}".format( - acl_names, perms - ) - expected = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": ", ".join(acl_names), - "acl_type": acl_type, - "perms": 7, - }, - "old": { - "acl_name": ", ".join(acl_names), - "acl_type": acl_type, - "perms": "A", - }, - }, - "result": None, - } - - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(ret, expected) - - # Update - test=False - with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Applied new permissions for {}".format(", ".join(acl_names)) - expected = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": ", ".join(acl_names), - "acl_type": acl_type, - "perms": "rwx", - } - }, - "result": True, - } - - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(expected, ret) - - # Update - modfacl error - with patch.dict( - linux_acl.__salt__, - { - "acl.modfacl": MagicMock( - side_effect=CommandExecutionError("Custom err") - ) - }, - ): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Error updating permissions for {}: Custom err".format( - acl_names - ) - expected = { - "name": name, - "comment": comt, - "changes": {}, - "result": False, - } - - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(expected, ret) - - # New - test=True - with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "New permissions will be applied for {}: {}".format( - acl_names, perms - ) - expected = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": ", ".join(acl_names), - "acl_type": acl_type, - "perms": perms, - } - }, - "result": None, - } - - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(expected, ret) - - # New - test=False - with patch.dict(linux_acl.__salt__, {"acl.modfacl": mock_modfacl}): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Applied new permissions for {}".format(", ".join(acl_names)) - expected = { - "name": name, - "comment": comt, - "changes": { - "new": { - "acl_name": ", ".join(acl_names), - "acl_type": acl_type, - "perms": perms, - } - }, - "result": True, - } - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(expected, ret) - - # New - modfacl error - with patch.dict( - linux_acl.__salt__, - { - "acl.modfacl": MagicMock( - side_effect=CommandExecutionError("Custom err") - ) - }, - ): - with patch.dict(linux_acl.__opts__, {"test": False}): - comt = "Error updating permissions for {}: Custom err".format( - acl_names - ) - expected = { - "name": name, - "comment": comt, - "changes": {}, - "result": False, - } - - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(expected, ret) - - # No acl type - comt = "ACL Type does not exist" - expected = { - "name": name, - "comment": comt, - "result": False, - "changes": {}, - } - ret = linux_acl.list_present(name, acl_type, acl_names, perms) - self.assertDictEqual(expected, ret) - - # 'list_absent' function tests: 2 - - def test_list_absent(self): - """ - Test to ensure a Linux ACL does not exist - """ - name = "/root" - acl_type = "users" - acl_names = ["damian", "homer"] - perms = "rwx" - - ret = {"name": name, "result": None, "comment": "", "changes": {}} - - mock = MagicMock( - side_effect=[ - { - name: { - acl_type: [ - {acl_names[0]: {"octal": "A"}, acl_names[1]: {"octal": "A"}} - ] - } - }, - {name: {acl_type: ""}}, - ] - ) - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Removing permissions" - ret.update({"comment": comt}) - self.assertDictEqual( - linux_acl.list_absent(name, acl_type, acl_names, perms), ret - ) - - comt = "ACL Type does not exist" - ret.update({"comment": comt, "result": False}) - self.assertDictEqual(linux_acl.list_absent(name, acl_type, acl_names), ret) - - def test_absent_recursive(self): - """ - Test to ensure a Linux ACL does not exist - """ - name = "/root" - acl_type = "users" - acl_name = "damian" - perms = "rwx" - - ret = {"name": name, "result": None, "comment": "", "changes": {}} - - mock = MagicMock( - side_effect=[ - { - name: {acl_type: [{acl_name: {"octal": 7}}]}, - name + "/foo": {acl_type: [{acl_name: {"octal": "A"}}]}, - } - ] - ) - with patch.dict(linux_acl.__salt__, {"acl.getfacl": mock}): - with patch.dict(linux_acl.__opts__, {"test": True}): - comt = "Removing permissions" - ret.update({"comment": comt}) - self.assertDictEqual( - linux_acl.absent(name, acl_type, acl_name, perms, recurse=True), ret - ) From b91f363951195cbb832cf18a726d51468663003f Mon Sep 17 00:00:00 2001 From: Frode Gundersen Date: Mon, 10 Apr 2023 11:38:53 -0600 Subject: [PATCH 028/152] Update tests/pytests/unit/states/test_linux_acl.py Co-authored-by: Pedro Algarvio --- tests/pytests/unit/states/test_linux_acl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pytests/unit/states/test_linux_acl.py b/tests/pytests/unit/states/test_linux_acl.py index 976a57b8c4b..60bbe55f51c 100644 --- a/tests/pytests/unit/states/test_linux_acl.py +++ b/tests/pytests/unit/states/test_linux_acl.py @@ -299,7 +299,6 @@ def test_list_present(): """ Test to ensure a Linux ACL is present """ - maxDiff = None name = "/root" acl_type = "user" acl_names = ["root", "damian", "homer"] From 6503765b3fcbe222db2830ca4d3aa92cfd0a5d02 Mon Sep 17 00:00:00 2001 From: jeanluc Date: Sat, 6 May 2023 23:57:52 +0200 Subject: [PATCH 029/152] Add test for issue 64232 --- .../integration/modules/test_x509_v2.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/pytests/integration/modules/test_x509_v2.py b/tests/pytests/integration/modules/test_x509_v2.py index 2fd005778c5..99d0d213397 100644 --- a/tests/pytests/integration/modules/test_x509_v2.py +++ b/tests/pytests/integration/modules/test_x509_v2.py @@ -673,6 +673,35 @@ def test_sign_remote_certificate_copypath(x509_salt_call_cli, cert_args, tmp_pat assert (tmp_path / f"{cert.serial_number:x}.crt").exists() +def test_create_private_key(x509_salt_call_cli): + """ + Ensure calling from the CLI works as expected and does not complain + about unknown internal kwargs (__pub_fun etc). + """ + ret = x509_salt_call_cli.run("x509.create_private_key") + assert ret.returncode == 0 + assert ret.data + assert ret.data.startswith("-----BEGIN PRIVATE KEY-----") + + +def test_create_crl(x509_salt_call_cli, ca_key, ca_cert, x509_pkidir): + """ + Ensure calling from the CLI works as expected and does not complain + about unknown internal kwargs (__pub_fun etc). + """ + with pytest.helpers.temp_file("key", ca_key, x509_pkidir) as ca_keyfile: + with pytest.helpers.temp_file("cert", ca_cert, x509_pkidir) as ca_certfile: + ret = x509_salt_call_cli.run( + "x509.create_crl", + revoked=[], + signing_private_key=str(ca_keyfile), + signing_cert=str(ca_certfile), + ) + assert ret.returncode == 0 + assert ret.data + assert ret.data.startswith("-----BEGIN X509 CRL-----") + + def _belongs_to(cert_or_pubkey, privkey): if isinstance(cert_or_pubkey, cx509.Certificate): cert_or_pubkey = cert_or_pubkey.public_key() From 57608c00679ef7acd328e664fd54eba141f9ed9d Mon Sep 17 00:00:00 2001 From: jeanluc Date: Sat, 6 May 2023 23:58:56 +0200 Subject: [PATCH 030/152] Fix x509_v2 unknown salt-internal kwargs --- changelog/64232.fixed.md | 1 + salt/modules/x509_v2.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 changelog/64232.fixed.md diff --git a/changelog/64232.fixed.md b/changelog/64232.fixed.md new file mode 100644 index 00000000000..45a5ccb90ea --- /dev/null +++ b/changelog/64232.fixed.md @@ -0,0 +1 @@ +Fixed x509_v2 `create_private_key`/`create_crl` unknown kwargs: __pub_fun... diff --git a/salt/modules/x509_v2.py b/salt/modules/x509_v2.py index b46d4cf57d7..0725b1b5624 100644 --- a/salt/modules/x509_v2.py +++ b/salt/modules/x509_v2.py @@ -901,8 +901,11 @@ def create_crl( salt.utils.versions.kwargs_warn_until(["text"], "Potassium") kwargs.pop("text") - if kwargs: - raise SaltInvocationError(f"Unrecognized keyword arguments: {list(kwargs)}") + unknown = [kwarg for kwarg in kwargs if not kwarg.startswith("_")] + if unknown: + raise SaltInvocationError( + f"Unrecognized keyword arguments: {list(unknown)}" + ) if days_valid is None: try: @@ -1235,8 +1238,9 @@ def create_private_key( for x in ignored_params: kwargs.pop(x) - if kwargs: - raise SaltInvocationError(f"Unrecognized keyword arguments: {list(kwargs)}") + unknown = [kwarg for kwarg in kwargs if not kwarg.startswith("_")] + if unknown: + raise SaltInvocationError(f"Unrecognized keyword arguments: {list(unknown)}") if encoding not in ["der", "pem", "pkcs12"]: raise CommandExecutionError( From aeaf55815ad09082ab1d9f9925b5732e0bce097b Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Wed, 26 Apr 2023 14:26:55 -0500 Subject: [PATCH 031/152] Call global logger when catching pip.list exceptions in states.pip.installed --- changelog/64169.fixed.md | 1 + salt/states/pip_state.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog/64169.fixed.md diff --git a/changelog/64169.fixed.md b/changelog/64169.fixed.md new file mode 100644 index 00000000000..499b94b693b --- /dev/null +++ b/changelog/64169.fixed.md @@ -0,0 +1 @@ +Call global logger when catching pip.list exceptions in states.pip.installed diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index 542a7f6c751..fd99d6bd626 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -852,7 +852,7 @@ def installed( ) # If we fail, then just send False, and we'll try again in the next function call except Exception as exc: # pylint: disable=broad-except - log.exception(exc) + globals().get("log").exception(exc) pip_list = False for prefix, state_pkg_name, version_spec in pkgs_details: From 3c552ecb907ee956250b0fd09bab96a34b3420af Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Wed, 26 Apr 2023 18:27:11 -0500 Subject: [PATCH 032/152] Add unit test for #64169 --- tests/pytests/unit/states/test_pip.py | 69 +++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/pytests/unit/states/test_pip.py diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py new file mode 100644 index 00000000000..7e04602ce44 --- /dev/null +++ b/tests/pytests/unit/states/test_pip.py @@ -0,0 +1,69 @@ +""" + :codeauthor: Eric Graham +""" +import logging + +import pytest + +import salt.states.pip_state as pip_state +from salt.exceptions import CommandExecutionError +from tests.support.mock import MagicMock, patch + + +@pytest.fixture +def configure_loader_modules(): + return { + pip_state: { + '__env__': 'base', + '__opts__': { + 'test': False + } + } + } + + +def test_issue_64169(caplog): + pkg_to_install = 'nonexistent_package' + exception_message = 'Invalid JSON (test_issue_64169)' + + mock_pip_list = MagicMock(side_effect=[ + CommandExecutionError(exception_message), # pre-cache the pip list (preinstall) + {}, # Checking if the pkg is already installed + {pkg_to_install: '100.10.1'} # Confirming successful installation + ]) + mock_pip_version = MagicMock(return_value='100.10.1') + mock_pip_install = MagicMock(return_value={"retcode": 0, "stdout": ""}) + + with patch.dict(pip_state.__salt__, { + "pip.list": mock_pip_list, + "pip.version": mock_pip_version, + "pip.install": mock_pip_install + }): + with caplog.at_level(logging.WARNING): + # Call pip.installed with a specifically 'broken' pip.list. + # pip.installed should continue, but log the exception from pip.list. + # pip.installed should NOT raise an exception itself. + # noinspection PyBroadException + try: + pip_state.installed( + name=pkg_to_install, + use_wheel=False, # Set False to simplify testing + no_use_wheel=False, # ' + no_binary=False, # ' + log=None # Regression will cause this function call to throw + # an AttributeError + ) + except AttributeError: + # Observed behavior in #64169 + assert False + except: + # Something went wrong, but it isn't what's being tested for here. + return + + # Take 64169 further and actually confirm that the targeted exception from pip.list got logged. + assert exception_message in caplog.messages + + # Confirm that the state continued to install the package as expected. + # Only check the 'pkgs' parameter of pip.install + mock_install_call_args, mock_install_call_kwargs = mock_pip_install.call_args + assert mock_install_call_kwargs['pkgs'] == pkg_to_install From 071a65fb10a72e23b9e22d11f7da6957b2c05f7c Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Mon, 1 May 2023 10:55:29 -0500 Subject: [PATCH 033/152] Rename Global Logger log to logger in pip_state.py --- changelog/64169.fixed.md | 1 + salt/states/pip_state.py | 14 ++++++++------ tests/pytests/unit/states/test_pip.py | 11 +++++++++-- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/changelog/64169.fixed.md b/changelog/64169.fixed.md index 499b94b693b..fe80eff1e94 100644 --- a/changelog/64169.fixed.md +++ b/changelog/64169.fixed.md @@ -1 +1,2 @@ Call global logger when catching pip.list exceptions in states.pip.installed +Rename gloabl logger `log` to `logger` inside pip_state \ No newline at end of file diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index fd99d6bd626..cc5d877c06e 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -114,7 +114,7 @@ if HAS_PIP is True: # pylint: enable=import-error -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) # Define the module's virtual name __virtualname__ = "pip" @@ -189,10 +189,10 @@ def _check_pkg_version_format(pkg): # vcs+URL urls are not properly parsed. # The next line is meant to trigger an AttributeError and # handle lower pip versions - log.debug("Installed pip version: %s", pip.__version__) + logger.debug("Installed pip version: %s", pip.__version__) install_req = _from_line(pkg) except AttributeError: - log.debug("Installed pip version is lower than 1.2") + logger.debug("Installed pip version is lower than 1.2") supported_vcs = ("git", "svn", "hg", "bzr") if pkg.startswith(supported_vcs): for vcs in supported_vcs: @@ -351,7 +351,7 @@ def _pep440_version_cmp(pkg1, pkg2, ignore_epoch=False): making the comparison. """ if HAS_PKG_RESOURCES is False: - log.warning( + logger.warning( "The pkg_resources packages was not loaded. Please install setuptools." ) return None @@ -367,7 +367,7 @@ def _pep440_version_cmp(pkg1, pkg2, ignore_epoch=False): if pkg_resources.parse_version(pkg1) > pkg_resources.parse_version(pkg2): return 1 except Exception as exc: # pylint: disable=broad-except - log.exception(exc) + logger.exception(f'Comparison of package versions "{pkg1}" and "{pkg2}" failed: {exc}') return None @@ -852,7 +852,9 @@ def installed( ) # If we fail, then just send False, and we'll try again in the next function call except Exception as exc: # pylint: disable=broad-except - globals().get("log").exception(exc) + logger.exception( + f'Pre-caching of PIP packages during states.pip.installed failed by exception from pip.list: {exc}' + ) pip_list = False for prefix, state_pkg_name, version_spec in pkgs_details: diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py index 7e04602ce44..1b6d8afb364 100644 --- a/tests/pytests/unit/states/test_pip.py +++ b/tests/pytests/unit/states/test_pip.py @@ -60,8 +60,15 @@ def test_issue_64169(caplog): # Something went wrong, but it isn't what's being tested for here. return - # Take 64169 further and actually confirm that the targeted exception from pip.list got logged. - assert exception_message in caplog.messages + # Take 64169 further and actually confirm that the exception from pip.list got logged. + exc_msg_present = False + for log_line in caplog.messages: + # The exception must be somewhere in the log, but may optionally not be on a line by itself. + if exception_message in log_line: + exc_msg_present = True + break + + assert exc_msg_present # Confirm that the state continued to install the package as expected. # Only check the 'pkgs' parameter of pip.install From a467c04d04695b7f61b530668736d84e6a3e1da8 Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Mon, 1 May 2023 15:51:25 -0500 Subject: [PATCH 034/152] Clarify Failing Test Message; Search for Entire Log Line in caplog --- tests/pytests/unit/states/test_pip.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py index 1b6d8afb364..7d93faa3eb8 100644 --- a/tests/pytests/unit/states/test_pip.py +++ b/tests/pytests/unit/states/test_pip.py @@ -53,22 +53,19 @@ def test_issue_64169(caplog): log=None # Regression will cause this function call to throw # an AttributeError ) - except AttributeError: + except AttributeError as exc: # Observed behavior in #64169 - assert False + pytest.fail( + 'Regression on #64169: pip_state.installed seems to be throwing an unexpected AttributeException: ' + f'{exc}' + ) except: # Something went wrong, but it isn't what's being tested for here. return # Take 64169 further and actually confirm that the exception from pip.list got logged. - exc_msg_present = False - for log_line in caplog.messages: - # The exception must be somewhere in the log, but may optionally not be on a line by itself. - if exception_message in log_line: - exc_msg_present = True - break - - assert exc_msg_present + assert 'Pre-caching of PIP packages during states.pip.installed failed by exception ' \ + f'from pip.list: {exception_message}' in caplog.messages # Confirm that the state continued to install the package as expected. # Only check the 'pkgs' parameter of pip.install From db1406a85fca0925f67a7989a1f58d8e928beb2b Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Mon, 1 May 2023 15:53:52 -0500 Subject: [PATCH 035/152] Fix Changelog Typo --- changelog/64169.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/64169.fixed.md b/changelog/64169.fixed.md index fe80eff1e94..d6ce2bf1937 100644 --- a/changelog/64169.fixed.md +++ b/changelog/64169.fixed.md @@ -1,2 +1,2 @@ Call global logger when catching pip.list exceptions in states.pip.installed -Rename gloabl logger `log` to `logger` inside pip_state \ No newline at end of file +Rename global logger `log` to `logger` inside pip_state \ No newline at end of file From 926270054d7b8694a2b17f18d2a924a65650832b Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Mon, 1 May 2023 16:08:55 -0500 Subject: [PATCH 036/152] Remove Silent Catch --- tests/pytests/unit/states/test_pip.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py index 7d93faa3eb8..a7cdc106e62 100644 --- a/tests/pytests/unit/states/test_pip.py +++ b/tests/pytests/unit/states/test_pip.py @@ -59,9 +59,6 @@ def test_issue_64169(caplog): 'Regression on #64169: pip_state.installed seems to be throwing an unexpected AttributeException: ' f'{exc}' ) - except: - # Something went wrong, but it isn't what's being tested for here. - return # Take 64169 further and actually confirm that the exception from pip.list got logged. assert 'Pre-caching of PIP packages during states.pip.installed failed by exception ' \ From 724fc208248c705dd472bdf5ce27992bed9f08cd Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Tue, 2 May 2023 13:22:13 -0500 Subject: [PATCH 037/152] Run Black Pre-Commit Step --- salt/states/pip_state.py | 16 ++++--- tests/pytests/unit/states/test_pip.py | 61 ++++++++++++++------------- 2 files changed, 40 insertions(+), 37 deletions(-) diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index cc5d877c06e..de75057adf4 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -251,7 +251,7 @@ def _check_if_installed( index_url, extra_index_url, pip_list=False, - **kwargs + **kwargs, ): """ Takes a package name and version specification (if any) and checks it is @@ -367,7 +367,9 @@ def _pep440_version_cmp(pkg1, pkg2, ignore_epoch=False): if pkg_resources.parse_version(pkg1) > pkg_resources.parse_version(pkg2): return 1 except Exception as exc: # pylint: disable=broad-except - logger.exception(f'Comparison of package versions "{pkg1}" and "{pkg2}" failed: {exc}') + logger.exception( + f'Comparison of package versions "{pkg1}" and "{pkg2}" failed: {exc}' + ) return None @@ -418,7 +420,7 @@ def installed( cache_dir=None, no_binary=None, extra_args=None, - **kwargs + **kwargs, ): """ Make sure the package is installed @@ -853,7 +855,7 @@ def installed( # If we fail, then just send False, and we'll try again in the next function call except Exception as exc: # pylint: disable=broad-except logger.exception( - f'Pre-caching of PIP packages during states.pip.installed failed by exception from pip.list: {exc}' + f"Pre-caching of PIP packages during states.pip.installed failed by exception from pip.list: {exc}" ) pip_list = False @@ -874,7 +876,7 @@ def installed( index_url, extra_index_url, pip_list, - **kwargs + **kwargs, ) # If _check_if_installed result is None, something went wrong with # the command running. This way we keep stateful output. @@ -980,7 +982,7 @@ def installed( no_cache_dir=no_cache_dir, extra_args=extra_args, disable_version_check=True, - **kwargs + **kwargs, ) if pip_install_call and pip_install_call.get("retcode", 1) == 0: @@ -1045,7 +1047,7 @@ def installed( user=user, cwd=cwd, env_vars=env_vars, - **kwargs + **kwargs, ) ) diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py index a7cdc106e62..307ba5e1e65 100644 --- a/tests/pytests/unit/states/test_pip.py +++ b/tests/pytests/unit/states/test_pip.py @@ -12,33 +12,33 @@ from tests.support.mock import MagicMock, patch @pytest.fixture def configure_loader_modules(): - return { - pip_state: { - '__env__': 'base', - '__opts__': { - 'test': False - } - } - } + return {pip_state: {"__env__": "base", "__opts__": {"test": False}}} def test_issue_64169(caplog): - pkg_to_install = 'nonexistent_package' - exception_message = 'Invalid JSON (test_issue_64169)' + pkg_to_install = "nonexistent_package" + exception_message = "Invalid JSON (test_issue_64169)" - mock_pip_list = MagicMock(side_effect=[ - CommandExecutionError(exception_message), # pre-cache the pip list (preinstall) - {}, # Checking if the pkg is already installed - {pkg_to_install: '100.10.1'} # Confirming successful installation - ]) - mock_pip_version = MagicMock(return_value='100.10.1') + mock_pip_list = MagicMock( + side_effect=[ + CommandExecutionError( + exception_message + ), # pre-cache the pip list (preinstall) + {}, # Checking if the pkg is already installed + {pkg_to_install: "100.10.1"}, # Confirming successful installation + ] + ) + mock_pip_version = MagicMock(return_value="100.10.1") mock_pip_install = MagicMock(return_value={"retcode": 0, "stdout": ""}) - with patch.dict(pip_state.__salt__, { - "pip.list": mock_pip_list, - "pip.version": mock_pip_version, - "pip.install": mock_pip_install - }): + with patch.dict( + pip_state.__salt__, + { + "pip.list": mock_pip_list, + "pip.version": mock_pip_version, + "pip.install": mock_pip_install, + }, + ): with caplog.at_level(logging.WARNING): # Call pip.installed with a specifically 'broken' pip.list. # pip.installed should continue, but log the exception from pip.list. @@ -47,24 +47,25 @@ def test_issue_64169(caplog): try: pip_state.installed( name=pkg_to_install, - use_wheel=False, # Set False to simplify testing + use_wheel=False, # Set False to simplify testing no_use_wheel=False, # ' - no_binary=False, # ' - log=None # Regression will cause this function call to throw - # an AttributeError + no_binary=False, # ' + log=None, # Regression will cause this function call to throw an AttributeError ) except AttributeError as exc: # Observed behavior in #64169 pytest.fail( - 'Regression on #64169: pip_state.installed seems to be throwing an unexpected AttributeException: ' - f'{exc}' + "Regression on #64169: pip_state.installed seems to be throwing an unexpected AttributeException: " + f"{exc}" ) # Take 64169 further and actually confirm that the exception from pip.list got logged. - assert 'Pre-caching of PIP packages during states.pip.installed failed by exception ' \ - f'from pip.list: {exception_message}' in caplog.messages + assert ( + "Pre-caching of PIP packages during states.pip.installed failed by exception " + f"from pip.list: {exception_message}" in caplog.messages + ) # Confirm that the state continued to install the package as expected. # Only check the 'pkgs' parameter of pip.install mock_install_call_args, mock_install_call_kwargs = mock_pip_install.call_args - assert mock_install_call_kwargs['pkgs'] == pkg_to_install + assert mock_install_call_kwargs["pkgs"] == pkg_to_install From 83cadc12f560b0c839fcb7dbbe8bcae46eb67c6c Mon Sep 17 00:00:00 2001 From: Eric Graham Date: Wed, 3 May 2023 09:39:26 -0500 Subject: [PATCH 038/152] Add New Line to Changelog --- changelog/64169.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/64169.fixed.md b/changelog/64169.fixed.md index d6ce2bf1937..e8631285aaa 100644 --- a/changelog/64169.fixed.md +++ b/changelog/64169.fixed.md @@ -1,2 +1,2 @@ Call global logger when catching pip.list exceptions in states.pip.installed -Rename global logger `log` to `logger` inside pip_state \ No newline at end of file +Rename global logger `log` to `logger` inside pip_state From bd57d085ad8b0d92e78ed15d701464179ce598ed Mon Sep 17 00:00:00 2001 From: ScriptAutomate Date: Wed, 10 May 2023 14:02:43 -0500 Subject: [PATCH 039/152] Update banners and links --- doc/_themes/saltstack2/layout.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/_themes/saltstack2/layout.html b/doc/_themes/saltstack2/layout.html index 04bff89e1fb..001844f7cd2 100644 --- a/doc/_themes/saltstack2/layout.html +++ b/doc/_themes/saltstack2/layout.html @@ -152,7 +152,7 @@ - +
- +
{% endif %} @@ -295,7 +295,7 @@ {% else %} {% endif %} #}--> - + {% if build_type=="next" %} From cb396fe805f31a779f8f4d47dd3e4e72a20ae9fc Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 19 May 2023 02:04:24 -0700 Subject: [PATCH 040/152] Ubuntu pkg tests run apt non-interactive mode. Issue #64307 --- pkg/tests/support/helpers.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/tests/support/helpers.py b/pkg/tests/support/helpers.py index 57b6ccd4d00..f4f26f0781a 100644 --- a/pkg/tests/support/helpers.py +++ b/pkg/tests/support/helpers.py @@ -596,8 +596,26 @@ class SaltPkgInstall: self.proc.run("launchctl", "disable", f"system/{service_name}") self.proc.run("launchctl", "bootout", "system", str(plist_file)) elif upgrade: + env = os.environ.copy() + extra_args = [] + if self.distro_id in ("ubuntu", "debian"): + env["DEBIAN_FRONTEND"] = "noninteractive" + extra_args = [ + "-o", + "DPkg::Options::=--force-confdef", + "-o", + "DPkg::Options::=--force-confold", + ] log.info("Installing packages:\n%s", pprint.pformat(self.pkgs)) - ret = self.proc.run(self.pkg_mngr, "upgrade", "-y", *self.pkgs) + args = extra_args + self.pkgs + ret = self.proc.run( + self.pkg_mngr, + "upgrade", + "-y", + *args, + _timeout=120, + env=env, + ) else: log.info("Installing packages:\n%s", pprint.pformat(self.pkgs)) ret = self.proc.run(self.pkg_mngr, "install", "-y", *self.pkgs) From 9dffea3178a0c183aafd322058e8bed8826441bd Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 22 May 2023 03:15:56 -0700 Subject: [PATCH 041/152] Check return code instead of stdout --- pkg/tests/support/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tests/support/helpers.py b/pkg/tests/support/helpers.py index f4f26f0781a..9853c441870 100644 --- a/pkg/tests/support/helpers.py +++ b/pkg/tests/support/helpers.py @@ -621,7 +621,7 @@ class SaltPkgInstall: ret = self.proc.run(self.pkg_mngr, "install", "-y", *self.pkgs) if not platform.is_darwin() and not platform.is_windows(): # Make sure we don't have any trailing references to old package file locations - assert "No such file or directory" not in ret.stdout + ret.returncode == 0 assert "/saltstack/salt/run" not in ret.stdout log.info(ret) self._check_retcode(ret) From eb71862449b93f496c678d892e0c0ad827278136 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 22 May 2023 16:35:31 +0100 Subject: [PATCH 042/152] Sometimes the first page does not have any results. Try next page if there's a next token. Signed-off-by: Pedro Algarvio --- tools/vm.py | 75 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/tools/vm.py b/tools/vm.py index 944f2fe6cc2..f7b2837ae1b 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -720,41 +720,50 @@ class VM: client = boto3.client("ec2", region_name=self.region_name) # Let's search for the launch template corresponding to this AMI launch_template_name = None + next_token = "" try: - response = response = client.describe_launch_templates( - Filters=[ - { - "Name": "tag:spb:is-golden-image-template", - "Values": ["true"], - }, - { - "Name": "tag:spb:project", - "Values": ["salt-project"], - }, - { - "Name": "tag:spb:environment", - "Values": [environment], - }, - { - "Name": "tag:spb:image-id", - "Values": [self.config.ami], - }, - ] - ) - log.debug( - "Search for launch template response:\n%s", pprint.pformat(response) - ) - for details in response.get("LaunchTemplates"): - if launch_template_name is not None: - log.warning( - "Multiple launch templates for the same AMI. This is not " - "supposed to happen. Picked the first one listed: %s", - response, - ) - break - launch_template_name = details["LaunchTemplateName"] + while True: + response = response = client.describe_launch_templates( + Filters=[ + { + "Name": "tag:spb:is-golden-image-template", + "Values": ["true"], + }, + { + "Name": "tag:spb:project", + "Values": ["salt-project"], + }, + { + "Name": "tag:spb:environment", + "Values": [environment], + }, + { + "Name": "tag:spb:image-id", + "Values": [self.config.ami], + }, + ], + NextToken=next_token, + ) + log.debug( + "Search for launch template response:\n%s", + pprint.pformat(response), + ) + for details in response.get("LaunchTemplates"): + if launch_template_name is not None: + log.warning( + "Multiple launch templates for the same AMI. This is not " + "supposed to happen. Picked the first one listed: %s", + response, + ) + break + launch_template_name = details["LaunchTemplateName"] - if launch_template_name is None: + if launch_template_name is not None: + break + + next_token = response.get("NextToken") + if next_token: + continue self.ctx.error(f"Could not find a launch template for {self.name!r}") self.ctx.exit(1) except ClientError as exc: From bd10d9444931024978ec788b73d51a40c6176ced Mon Sep 17 00:00:00 2001 From: natalieswork Date: Mon, 22 May 2023 16:07:26 -0400 Subject: [PATCH 043/152] removing references to Azure from existing files --- DEPENDENCIES.md | 335 +++++------ doc/conf.py | 1 - doc/ref/clouds/all/index.rst | 2 - doc/ref/configuration/master.rst | 23 - doc/ref/file_server/all/index.rst | 1 - doc/ref/grains/all/index.rst | 1 - doc/ref/modules/all/index.rst | 4 - doc/ref/pillar/all/index.rst | 1 - doc/ref/states/all/index.rst | 4 - doc/spelling_wordlist.txt | 1 - doc/topics/cloud/cloud.rst | 8 +- doc/topics/cloud/index.rst | 2 - requirements/static/ci/common.in | 1 - requirements/static/ci/py3.10/cloud.txt | 432 +------------- requirements/static/ci/py3.10/darwin.txt | 422 -------------- requirements/static/ci/py3.10/freebsd.txt | 410 -------------- requirements/static/ci/py3.10/lint.txt | 409 -------------- requirements/static/ci/py3.10/linux.txt | 411 -------------- .../static/ci/py3.11/darwin-crypto.txt | 10 + requirements/static/ci/py3.11/darwin.txt | 422 -------------- .../static/ci/py3.11/freebsd-crypto.txt | 10 + requirements/static/ci/py3.11/freebsd.txt | 410 -------------- .../static/ci/py3.11/linux-crypto.txt | 10 + requirements/static/ci/py3.11/linux.txt | 410 -------------- .../static/ci/py3.11/windows-crypto.txt | 10 + requirements/static/ci/py3.7/cloud.txt | 432 +------------- requirements/static/ci/py3.7/freebsd.txt | 514 ++--------------- requirements/static/ci/py3.7/lint.txt | 510 +---------------- requirements/static/ci/py3.7/linux.txt | 529 ++---------------- requirements/static/ci/py3.8/cloud.txt | 432 +------------- requirements/static/ci/py3.8/freebsd.txt | 497 ++-------------- requirements/static/ci/py3.8/lint.txt | 489 +--------------- requirements/static/ci/py3.8/linux.txt | 512 ++--------------- requirements/static/ci/py3.9/cloud.txt | 432 +------------- requirements/static/ci/py3.9/darwin.txt | 422 -------------- requirements/static/ci/py3.9/freebsd.txt | 490 +--------------- requirements/static/ci/py3.9/lint.txt | 476 +--------------- requirements/static/ci/py3.9/linux.txt | 501 ++--------------- salt/config/__init__.py | 3 - 39 files changed, 476 insertions(+), 9513 deletions(-) create mode 100644 requirements/static/ci/py3.11/darwin-crypto.txt create mode 100644 requirements/static/ci/py3.11/freebsd-crypto.txt create mode 100644 requirements/static/ci/py3.11/linux-crypto.txt create mode 100644 requirements/static/ci/py3.11/windows-crypto.txt diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index 2daccac12a9..849b1ed5e33 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -1,210 +1,125 @@ -| **OSS Software Name** | **Version** | **Primary License** | **Source Code Download URL** | **Author** | **Copyright Year** | -| --- | :--- | --- | --- | --- | ---: | -| | | | | | | -| Cheetah3 | 3.1.0 | MIT/X11 | https://pypi.org/project/Cheetah3/ | Travis Rudd | 2017-2019 | -| CherryPy | 17.3.0 | BSD | https://pypi.org/project/CherryPy/ | CherryPy Team | 2004-2019 | -| Genshi | 0.7.3 | BSD | https://pypi.org/project/Genshi/ | Edgewall Software | 2006-2010 | -| GitPython | 3.1.3 | BSD | https://pypi.org/project/GitPython/ | Sebastian Thiel, Michael Trier | 2008, 2009 | -| Jinja2 | 2.11.2 | BSD | https://pypi.org/project/Jinja2 | Armin Ronacher | 2007 | -| Mako | 1.1.3 | MIT/X11 | https://pypi.org/project/Mako/ | Mike Bayer | 2006-2020 | -| MarkupSafe | 1.1.1 | BSD | https://pypi.org/project/MarkupSafe/ | Armin Ronacher | 2010 | -| PyJWT | 1.7.1 | MIT/X11 | https://pypi.org/project/PyJWT/ | Jose Padilla | 2015 | -| PyNaCl | 1.4.0 | Apache License, V2.0 | https://pypi.org/project/PyNaCl/ | The PyNaCl developers | 2004 | -| PyYAML | 5.3.1 | MIT/X11 | https://pypi.org/project/PyYAML/ | Kirill Simonov | 2006-2019 | -| WerkZeug | 1.0.1 | BSD | https://pypi.org/project/Werkzeug/ | Armin Ronacher | 2007 | -| adal | 1.2.4 | MIT/X11 | https://pypi.org/project/adal | Microsoft Corporation | 2015 | -| apache-libcloud | 2.0.0 | Apache License, V2.0 | https://pypi.org/project/apache-libcloud/ | Apache Software Foundation | 2004 | -| appdirs | 1.4.4 | MIT/X11 | https://pypi.org/project/appdirs/ | Trent Mick | 2010 | -| asn1crypto | 1.3.0 | MIT/X11 | https://pypi.org/project/asn1crypto/ | wbond | 2015-2019 | -| attrs | 19.3.1 | MIT/X11 | https://pypi.org/project/attrs/ | Hynek Schlawack | 2015 | -| aws-sam-translator | 1.25.0 | Apache License, V2.0 | https://pypi.org/project/aws-sam-translator/ | Amazon Web Services | 2004 | -| aws-xray-sdk | 2.6.0 | Apache License, V2.0 | https://pypi.org/project/aws-xray-sdk/ | Amazon Web Services | 2004 | -| azure | 4.0.0 | MIT/X11 | https://pypi.org/project/azure | Microsoft Corporation | 2016 | -| azure-applicationinsights | 0.1.0 | MIT/X11 | https://pypi.org/project/azure-applicationinsights/ | Microsoft Corporation | 2016 | -| azure-batch | 4.1.3 | MIT/X11 | https://pypi.org/project/azure-batch/ | Microsoft Corporation | 2016 | -| azure-common | 1.1.25 | MIT/X11 | https://pypi.org/project/azure-common/ | Microsoft Corporation | 2016 | -| azure-cosmosdb-nspkg | 2.0.2 | Apache License, V2.0 | https://pypi.org/project/azure-cosmosdb-nspkg | Microsoft Corporation | 2004 | -| azure-cosmosdb-table | 1.0.6 | Apache License, V2.0 | https://pypi.org/project/azure-cosmosdb-table/ | Microsoft Corporation | 2004 | -| azure-datalake-store | 0.0.48 | MIT/X11 | https://pypi.org/project/azure-datalake-store/ | Microsoft Corporation | 2016 | -| azure-eventgrid | 1.3.0 | MIT/X11 | https://pypi.org/project/azure-eventgrid/ | Microsoft Corporation | 2016 | -| azure-graphrbac | 0.40.0 | MIT/X11 | https://pypi.org/project/azure-graphrbac/ | Microsoft Corporation | 2016 | -| azure-keyvault | 1.1.0 | MIT/X11 | https://pypi.org/project/azure-keyvault/ | Microsoft Corporation | 2016 | -| azure-loganalytics | 0.1.0 | MIT/X11 | https://pypi.org/project/azure-loganalytics/ | Microsoft Corporation | 2016 | -| azure-mgmt | 4.0.0 | MIT/X11 | https://pypi.org/project/azure-mgmt/ | Microsoft Corporation | 2016 | -| azure-mgmt-advisor | 1.0.1 | MIT/X11 | https://pypi.org/project/azure-mgmt-advisor/ | Microsoft Corporation | 2016 | -| azure-mgmt-applicationinsights | 0.1.1 | MIT/X11 | https://pypi.org/project/azure-mgmt-applicationinsights/ | Microsoft Corporation | 2016 | -| azure-mgmt-authorization | 0.50.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-authorization/ | Microsoft Corporation | 2016 | -| azure-mgmt-batch | 5.0.1 | MIT/X11 | https://pypi.org/project/azure-mgmt-batch/ | Microsoft Corporation | 2016 | -| azure-mgmt-batchai | 2.0.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-batchai/ | Microsoft Corporation | 2016 | -| azure-mgmt-billing | 0.2.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-billing/ | Microsoft Corporation | 2016 | -| azure-mgmt-cdn | 3.1.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-cdn/ | Microsoft Corporation | 2016 | -| azure-mgmt-cognitiveservices | 3.0.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-cognitiveservices/ | Microsoft Corporation | 2016 | -| azure-mgmt-commerce | 1.0.1 | MIT/X11 | https://pypi.org/project/azure-mgmt-commerce/ | Microsoft Corporation | 2016 | -| azure-mgmt-compute | 4.6.2 | MIT/X11 | https://pypi.org/project/azure-mgmt-compute/ | Microsoft Corporation | 2016 | -| azure-mgmt-consumption | 2.0.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-consumption/ | Microsoft Corporation | 2016 | -| azure-mgmt-containerinstance | 1.5.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-containerinstance/ | Microsoft Corporation | 2016 | -| azure-mgmt-containerregistry | 2.8.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-containerregistry/ | Microsoft Corporation | 2016 | -| azure-mgmt-containerservice | 4.4.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-containerservice/ | Microsoft Corporation | 2016 | -| azure-mgmt-cosmosdb | 0.4.1 | MIT/X11 | https://pypi.org/project/azure-mgmt-cosmosdb/ | Microsoft Corporation | 2016 | -| azure-mgmt-datafactory | 0.6.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-datafactory/ | Microsoft Corporation | 2016 | -| azure-mgmt-datalake-analytics | 0.6.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-datalake-analytics/ | Microsoft Corporation | 2016 | -| azure-mgmt-datalake-nspkg | 3.0.1 | MIT/X11 | https://pypi.org/project/azure-mgmt-datalake-nspkg/ | Microsoft Corporation | 2016 | -| azure-mgmt-datalake-store | 0.5.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-datalake-store/ | Microsoft Corporation | 2016 | -| azure-mgmt-datamigration | 1.0.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-datamigration/ | Microsoft Corporation | 2016 | -| azure-mgmt-devspaces | 0.1.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-devspaces/ | Microsoft Corporation | 2016 | -| azure-mgmt-devtestlabs | 2.2.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-devtestlabs/ | Microsoft Corporation | 2016 | -| azure-mgmt-dns | 2.1.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-dns/ | Microsoft Corporation | 2016 | -| azure-mgmt-eventgrid | 1.0.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-eventgrid/ | Microsoft Corporation | 2016 | -| azure-mgmt-eventhub | 2.6.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-eventhub/ | Microsoft Corporation | 2016 | -| azure-mgmt-hanaonazure | 0.1.1 | MIT/X11 | https://pypi.org/project/azure-mgmt-hanaonazure/ | Microsoft Corporation | 2016 | -| azure-mgmt-iotcentral | 0.1.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-iotcentral/ | Microsoft Corporation | 2016 | -| azure-mgmt-iothub | 0.5.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-iothub/ | Microsoft Corporation | 2016 | -| azure-mgmt-iothubprovisioningservices | 0.2.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-iothubprovisioningservices/ | Microsoft Corporation | 2016 | -| azure-mgmt-keyvault | 1.1.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-keyvault/ | Microsoft Corporation | 2016 | -| azure-mgmt-loganalytics | 0.2.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-loganalytics/ | Microsoft Corporation | 2016 | -| azure-mgmt-logic | 3.0.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-logic/ | Microsoft Corporation | 2016 | -| azure-mgmt-machinelearningcompute | 0.4.1 | MIT/X11 | https://pypi.org/project/azure-mgmt-machinelearningcompute/ | Microsoft Corporation | 2016 | -| azure-mgmt-managementgroups | 0.1.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-managementgroups/ | Microsoft Corporation | 2016 | -| azure-mgmt-managementpartner | 0.1.1 | MIT/X11 | https://pypi.org/project/azure-mgmt-managementpartner/ | Microsoft Corporation | 2016 | -| azure-mgmt-maps | 0.1.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-maps/ | Microsoft Corporation | 2016 | -| azure-mgmt-marketplaceordering | 0.1.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-marketplaceordering/ | Microsoft Corporation | 2016 | -| azure-mgmt-media | 1.0.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-media/ | Microsoft Corporation | 2016 | -| azure-mgmt-monitor | 0.5.2 | MIT/X11 | https://pypi.org/project/azure-mgmt-monitor/ | Microsoft Corporation | 2016 | -| azure-mgmt-msi | 0.2.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-msi/ | Microsoft Corporation | 2016 | -| azure-mgmt-network | 2.7.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-network/ | Microsoft Corporation | 2016 | -| azure-mgmt-notificationhubs | 2.1.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-notificationhubs/ | Microsoft Corporation | 2016 | -| azure-mgmt-nspkg | 3.0.2 | MIT/X11 | https://pypi.org/project/azure-mgmt-nspkg/ | Microsoft Corporation | 2016 | -| azure-mgmt-policyinsights | 0.1.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-policyinsights/ | Microsoft Corporation | 2016 | -| azure-mgmt-powerbiembedded | 2.0.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-powerbiembedded/ | Microsoft Corporation | 2016 | -| azure-mgmt-rdbms | 1.9.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-rdbms/ | Microsoft Corporation | 2016 | -| azure-mgmt-recoveryservices | 0.3.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-recoveryservices/ | Microsoft Corporation | 2016 | -| azure-mgmt-recoveryservicesbackup | 0.3.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-recoveryservicesbackup/ | Microsoft Corporation | 2016 | -| azure-mgmt-redis | 5.0.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-redis/ | Microsoft Corporation | 2016 | -| azure-mgmt-relay | 0.1.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-relay/ | Microsoft Corporation | 2016 | -| azure-mgmt-reservations | 0.2.1 | MIT/X11 | https://pypi.org/project/azure-mgmt-reservations/ | Microsoft Corporation | 2016 | -| azure-mgmt-resource | 2.2.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-resource/ | Microsoft Corporation | 2016 | -| azure-mgmt-scheduler | 2.0.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-scheduler/ | Microsoft Corporation | 2016 | -| azure-mgmt-search | 2.1.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-search/ | Microsoft Corporation | 2016 | -| azure-mgmt-servicebus | 0.5.3 | MIT/X11 | https://pypi.org/project/azure-mgmt-servicebus/ | Microsoft Corporation | 2016 | -| azure-mgmt-servicefabric | 0.2.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-servicefabric/ | Microsoft Corporation | 2016 | -| azure-mgmt-signalr | 0.1.1 | MIT/X11 | https://pypi.org/project/azure-mgmt-signalr/ | Microsoft Corporation | 2016 | -| azure-mgmt-sql | 0.9.1 | MIT/X11 | https://pypi.org/project/azure-mgmt-sql/ | Microsoft Corporation | 2016 | -| azure-mgmt-storage | 2.0.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-storage/ | Microsoft Corporation | 2016 | -| azure-mgmt-subscription | 0.2.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-subscription/ | Microsoft Corporation | 2016 | -| azure-mgmt-trafficmanager | 0.50.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-trafficmanager/ | Microsoft Corporation | 2016 | -| azure-mgmt-web | 0.35.0 | MIT/X11 | https://pypi.org/project/azure-mgmt-web/ | Microsoft Corporation | 2016 | -| azure-nspkg | 3.0.2 | MIT/X11 | https://pypi.org/project/azure-nspkg/ | Microsoft Corporation | 2016 | -| azure-servicebus | 0.21.1 | MIT/X11 | https://pypi.org/project/azure-servicebus/ | Microsoft Corporation | 2016 | -| azure-servicefabric | 6.3.0.0 | MIT/X11 | https://pypi.org/project/azure-servicefabric/ | Microsoft Corporation | 2016 | -| azure-servicemanagement-legacy | 0.20.7 | Apache License, V2.0 | https://pypi.org/project/azure-servicemanagement-legacy/ | Microsoft Corporation | 2016 | -| azure-storage-blob | 1.5.0 | MIT/X11 | https://pypi.org/project/azure-storage-blob/ | Microsoft Corporation | 2017 | -| azure-storage-common | 1.4.2 | MIT/X11 | https://pypi.org/project/azure-storage-common/ | Microsoft Corporation | 2017 | -| azure-storage-file | 1.4.0 | MIT/X11 | https://pypi.org/project/azure-storage-file/ | Microsoft Corporation | 2016 | -| azure-storage-queue | 1.4.0 | MIT/X11 | https://pypi.org/project/azure-storage-queue/ | Microsoft Corporation | 2017 | -| bcrypt | 3.1.7 | Apache License, V2.0 | https://pypi.org/project/bcrypt/ | The Python Cryptographic Authority | 2013 | -| boto | 2.49.0 | MIT/X11 | https://pypi.org/project/boto/ | Mitch Garnaatt | 2013 | -| boto3 | 1.14.16 | Apache License, V2.0 | https://pypi.org/project/boto3/ | AWS | 2019 | -| botocore | 1.17.16 | Apache License, V2.0 | https://pypi.org/project/botocore/ | AWS | 2019 | -| cachetools | 4.1.1 | MIT/X11 | https://pypi.org/project/cachetools/ | tkem | 2014-2020 | -| cassandra-driver | 3.24.0 | Apache License, V2.0 | https://pypi.org/project/cassandra-driver/ | DataStax | 2013 | -| certifi | 2020.6.20 | Mozilla Public License, V2.0 | https://pypi.org/project/certifi/ | Kenneth Reitz | 2013 | -| certvalidator | 0.11.1 | MIT/X11 | https://pypi.org/project/certvalidator/ | wbond | 2015-2018 | -| cffi | 1.14.0 | MIT/X11 | https://pypi.org/project/cffi/ | Armin Rigo, Maciej Fijalkowski | 2005-2007 | -| cfn-lint | 0.33.2 | MIT/X11 | https://pypi.org/project/cfn-lint/ | kddejong | 2018 | -| chardet | 3.0.4 | GNU Lesser General Public License, V2.1 | https://pypi.org/project/chardet/ | Daniel Blanchard | 2012 | -| cheroot | 8.3.0 | BSD | https://pypi.org/project/cheroot/ | CherryPy Team | 2004-2020 | -| click | 7.1.2 | BSD | https://pypi.org/project/click/ | Pallets | 2014 | -| contextlib2 | 0.6.0.post1 | Python License | https://pypi.org/project/contextlib2/ | Nick Coghlan | 2015 | -| croniter | 0.3.34 | MIT/X11 | https://pypi.org/project/croniter/ | Matsumoto Taichi, kiorky | 2014 | -| cryptography | 2.9.2 | BSD | https://pypi.org/project/cryptography/ | The cryptography developers | 2014 | -| decorator | 4.4.2 | BSD | https://pypi.org/project/decorator/ | Michele Simionato | 2005-2018 | -| distlib | 0.3.1 | Python License | https://pypi.org/project/distlib/ | Vinay Sajip | 2012 | -| dnspython | 1.16.0 | BSD | https://pypi.org/project/dnspython/ | Bob Halley | 2001-2017 | -| docker | 4.2.2 | Apache License, V2.0 | https://pypi.org/project/docker/ | Joffery F | 2016 | -| docutils | 0.15.2 | BSD | https://pypi.org/project/docutils/ | David Goodger | 2011 | -| ecdsa | 0.15 | MIT/X11 | https://pypi.org/project/ecdsa/ | Brian Warner | 2010 | -| filelock | 3.0.12 | Public Domain | https://pypi.org/project/filelock/ | Benedikt Schmitt | 2018 | -| future | 0.18.2 | MIT/X11 | https://pypi.org/project/future/ | Ed Schofield | 2013-2019 | -| geomet | 0.2.1.post1 | Apache License, V2.0 | https://pypi.org/project/geomet/ | Lars Butler | 2014 | -| gitdb | 4.0.5 | BSD | https://pypi.org/project/gitdb/ | Sebastian Thiel | 2010, 2011 | -| google-auth | 1.18.0 | Apache License, V2.0 | https://pypi.org/project/google-auth/ | Google Cloud Platform | 2016 | -| hgtools | 8.2.0 | MIT/X11 | https://pypi.org/project/hgtools/ | Jason R. Coombs | 2019 | -| idna | 2.8 | BSD | https://pypi.org/project/idna/ | Kim Davies | 2013-2020 | -| importlib-metadata | 1.7.0 | Apache License, V2.0 | https://pypi.org/project/importlib-metadata/ | Barry Warsaw | 2017-2019 | -| ipaddress | 1.0.23 | Python License | https://pypi.org/project/ipaddress/ | Philipp Hagemeister | 2014 | -| isodate | 0.6.0 | BSD | https://pypi.org/project/isodate/ | Gerhard Weis | 2009 | -| jaraco.functools | 3.0.1 | MIT/X11 | https://pypi.org/project/jaraco.functools/ | Jason R. Coombs | 2020 | -| jmespath | 0.10.0 | MIT/X11 | https://pypi.org/project/jmespath/ | James Saryerwinnie | 2013 | -| jsondiff | 1.1.2 | MIT/X11 | https://pypi.org/project/jsondiff/ | Zoomer Analytics LLC | 2015 | -| jsonpatch | 1.26 | BSD | https://pypi.org/project/jsonpatch/ | Stefan Kögl | 2011 | -| jsonpickle | 1.4.1 | BSD | https://pypi.org/project/jsonpickle/ | David Aguilar | 2009-2018 | -| jsonpointer | 2 | BSD | https://pypi.org/project/jsonpointer/ | Stefan Kögl | 2017 | -| jsonschema | 3.2.0 | MIT/X11 | https://pypi.org/project/jsonschema/ | Julian Berman | 2013 | -| junit-xml | 1.9 | MIT/X11 | https://pypi.org/project/junit-xml/ | Brian Beyer | 2013 | -| junos-eznc | 2.4.0 | Apache License, V2.0 | https://pypi.org/project/junos-eznc/ | Jeremy Schulman, Nitin Kumar, Rick Sherman, Stacy Smith | 2018 | -| jxmlease | 1.0.3 | MIT/X11 | https://pypi.org/project/jxmlease/ | Juniper Networks | 2015-2016 | -| kazoo | 2.7.0 | Apache License, V2.0 | https://pypi.org/project/kazoo/ | Kazoo team | 2012 | -| keyring | 5.7.1 | MIT/X11 | https://pypi.org/project/keyring/ | Kang Zhang | 2019 | -| kubernetes | 3.0.0 | Apache License, V2.0 | https://pypi.org/project/kubernetes/ | Kubernetes | 2016 | -| libnacl | 1.7.1 | Apache License, V2.0 | https://pypi.org/project/libnacl/ | Thomas S Hatch | 2014 | -| lxml | 4.5.1 | BSD | https://pypi.org/project/lxml/ | lxml dev team | 2019 | -| mock | 4.0.2 | BSD | https://pypi.org/project/mock/ | Testing Cabal | 2003-2013 | -| more-itertools | 5.0.0 | MIT/X11 | https://pypi.org/project/more-itertools/ | Eric Rose | 2012 | -| moto | 1.3.14 | Apache License, V2.0 | https://pypi.org/project/moto/ | Steve Pulec | 2004 | -| msrest | 0.6.17 | MIT/X11 | https://pypi.org/project/msrest/ | Microsoft | 2016 | -| msrestazure | 0.6.4 | MIT/X11 | https://pypi.org/project/msrestazure/ | Microsoft | 2016 | -| natsort | 7.0.1 | MIT/X11 | https://pypi.org/project/natsort/ | Seth M. Morton | 2012-2020 | -| ncclient | 0.6.7 | Apache License, V2.0 | https://pypi.org/project/ncclient/ | Shikhar Bhushan, Leonidas Poulopoulos, Ebben Aries, Einar Nilsen-Nygaard | 2004 | -| netaddr | 0.8.0 | BSD | https://pypi.org/project/netaddr/ | Author: David P. D. Moss, Stefan Nordhausen et al | 2008 | -| networkx | 2.4 | BSD | https://pypi.org/project/networkx/ | Aric Hagberg | 2004-2020 | -| ntc-templates | 1.5.0 | Apache License, V2.0 | https://pypi.org/project/ntc-templates/ | Jason Edelman | 2015 | -| oauthlib | 3.1.0 | BSD | https://pypi.org/project/oauthlib/ | The OAuthlib Community | 2019 | -| oscrypto | 1.2.0 | MIT/X11 | https://pypi.org/project/oscrypto/ | wbond | 2015-2019 | -| paramiko | 2.7.1 | GNU Lesser General Public License, V2.1 | https://pypi.org/project/paramiko/ | Jeff Forcier | 1999 | -| passlib | 1.7.2 | BSD | https://pypi.org/project/passlib/ | Eli Collins | 2004 | -| pathtools | 0.1.2 | MIT/X11 | https://pypi.org/project/pathtools/ | Yesudeep Mangalapilly | 2010 | -| portend | 2.6 | MIT/X11 | https://pypi.org/project/portend/ | Jason R Coombs | 2018 | -| psutil | 5.7.0 | BSD | https://pypi.org/project/psutil/ | Giampaolo Rodola | 2009 | -| pyOpenSSL | 19.1.0 | Apache License, V2.0 | https://pypi.org/project/pyOpenSSL/ | The pyopenssl Developers | 2004 | -| pyasn1 | 0.4.8 | BSD | https://pypi.org/project/pyasn1/ | Ilya Etingof | 2005-2020 | -| pyasn1-modules | 0.2.8 | BSD | https://pypi.org/project/pyasn1-modules/ | Ilya Etingof | 2005-2020 | -| pycparser | 2.20 | BSD | https://pypi.org/project/pycparser/ | Eli Bendersky | 2008-2017 | -| pygit2 | 1.2.1 | GNU General Public License, V2.0 | https://pypi.org/project/pygit2/ | J. David Ibáñez | 2012 | -| pyinotify | 0.9.6 | MIT/X11 | https://pypi.org/project/pyinotify/ | Sebastien Martini | 2010 | -| pyparsing | 2.4.7 | MIT/X11 | https://pypi.org/project/pyparsing/ | Paul McGuire | 2018 | -| pyrsistent | 0.16.0 | MIT/X11 | https://pypi.org/project/pyrsistent/ | Tobias Gustafsson | 2019 | -| pyserial | 3.4 | BSD | https://pypi.org/project/pyserial/ | Chris Liechti | 2001-2016 | -| python-dateutil | 2.8.1 | Apache License, V2.0 | https://pypi.org/project/python-dateutil/ | Gustavo Niemeyer | 2017 | -| python-etcd | 0.4.5 | MIT/X11 | https://pypi.org/project/python-etcd/ | Jose Plana | 2015 | -| python-gnupg | 0.4.6 | BSD | https://pypi.org/project/python-gnupg/ | Vinay Sajip | 2008-2019 | -| python-jose[cryptography] | 3.2.0 | MIT/X11 | https://pypi.org/project/python-jose/ | Michael Davis | 2015 | -| pytz | 2020.1 | MIT/X11 | https://pypi.org/project/pytz/ | Stuart Bishop | 2015 | -| pyvmomi | 7 | Apache License, V2.0 | https://pypi.org/project/pyvmomi/ | VMware, Inc. | 2004 | -| requests | 2.24.0 | Apache License, V2.0 | https://pypi.org/project/requests/ | Kenneth Reitz | 2004 | -| requests-oauthlib | 1.3.0 | BSD | https://pypi.org/project/requests-oauthlib/ | Kenneth Reitz | 2014 | -| responses | 0.10.15 | Apache License, V2.0 | https://pypi.org/project/responses/ | David Cramer | 2004 | -| rfc3987 | 1.3.8 | GNU General Public License, V3.0 | https://pypi.org/project/rfc3987/ | Daniel Gerber | 2007 | -| rsa | 4.6 | Apache License, V2.0 | https://pypi.org/project/rsa/ | Sybren A. Stuvel | 2011 | -| s3transfer | 0.3.3 | Apache License, V2.0 | https://pypi.org/project/s3transfer/ | Amazon Web Services | 2004 | -| scp | 0.13.2 | GNU Lesser General Public License, V2.1 | https://pypi.org/project/scp/ | James Bardin | 2013 | -| setproctitle | 1.1.10 | BSD | https://pypi.org/project/setproctitle/ | Daniele Varrazzo | 2009-2020 | -| setuptools-scm | 4.1.2 | MIT/X11 | https://pypi.org/project/setuptools-scm/ | Ronny Pfannschmidt | 2010 | -| six | 1.15.0 | MIT/X11 | https://pypi.org/project/six/ | Benjamin Peterson | 2010-2020 | -| smmap | 3.0.4 | BSD | https://pypi.org/project/smmap/ | Sebastian Thiel | 2010, 2011 | -| sshpubkeys | 3.1.0 | BSD | https://pypi.org/project/sshpubkeys/ | Olli Jarva | 2014 | -| strict-rfc3339 | 0.7 | GNU General Public License, V3.0 | https://pypi.org/project/strict-rfc3339/ | Daniel Richman, Adam Greig | 2007 | -| tempora | 3.0.0 | MIT/X11 | https://pypi.org/project/tempora/ | Jason R. Coombs | 2020 | -| textfsm | 1.1.0 | Apache License, V2.0 | https://pypi.org/project/textfsm/ | Google | 2004 | -| timelib | 0.2.4 | BSD | https://pypi.org/project/timelib/ | Ralf Schmitt | 2009-2011 | -| toml | 0.10.1 | MIT/X11 | https://pypi.org/project/toml/ | William Pearson | 2013-2019 | -| transitions | 0.8.2 | MIT/X11 | https://pypi.org/project/transitions/ | Tal Yarkoni | 2014 - 2020 | -| urllib3 | 1.25.9 | MIT/X11 | https://pypi.org/project/urllib3/ | Andrey Petrov | 2008-2020 | -| vcert | 0.7.4 | ASL | https://pypi.org/project/vcert | Denis Subbotin | 2018 | -| virtualenv | 20.0.25 | MIT | https://pypi.org/project/virtualenv | Bernat Gabor | 2020-202x | -| watchdog | 0.10.3 | Apache License 2.0 | https://pypi.org/project/watchdog | Yesudeep Mangalapilly | 2012 | -| websocket-client | 0.57.0 | BSD | https://pypi.org/project/websocket-client | liris | 2018 | -| wrapt | 1.12.1 | BSD | https://pypi.org/project/wrapt | Graham Dumpleton | 2013-2019 | -| xmltodict | 0.12.0 | MIT | https://pypi.org/project/xmltodict | Martin Blech | 2010 | -| yamlordereddictloader | 0.4.0 | MIT License | https://pypi.org/project/yamlordereddictloader | François Ménabé | 2017 | -| zc.lockfile | 2 | ZPL 2.1 | https://pypi.org/project/zc.lockfile | Zope Foundation | 2010 | -| zipp | 3.1.0 | MIT | https://pypi.org/project/zipp | Jason R. Coombs | 2019 | +| **OSS Software Name** | **Version** | **Primary License** | **Source Code Download URL** | **Author** | **Copyright Year** | +| ------------------------- | :---------- | --------------------------------------- | ---------------------------------------------- | ------------------------------------------------------------------------ | -----------------: | +| | | | | | | +| Cheetah3 | 3.1.0 | MIT/X11 | https://pypi.org/project/Cheetah3/ | Travis Rudd | 2017-2019 | +| CherryPy | 17.3.0 | BSD | https://pypi.org/project/CherryPy/ | CherryPy Team | 2004-2019 | +| Genshi | 0.7.3 | BSD | https://pypi.org/project/Genshi/ | Edgewall Software | 2006-2010 | +| GitPython | 3.1.3 | BSD | https://pypi.org/project/GitPython/ | Sebastian Thiel, Michael Trier | 2008, 2009 | +| Jinja2 | 2.11.2 | BSD | https://pypi.org/project/Jinja2 | Armin Ronacher | 2007 | +| Mako | 1.1.3 | MIT/X11 | https://pypi.org/project/Mako/ | Mike Bayer | 2006-2020 | +| MarkupSafe | 1.1.1 | BSD | https://pypi.org/project/MarkupSafe/ | Armin Ronacher | 2010 | +| PyJWT | 1.7.1 | MIT/X11 | https://pypi.org/project/PyJWT/ | Jose Padilla | 2015 | +| PyNaCl | 1.4.0 | Apache License, V2.0 | https://pypi.org/project/PyNaCl/ | The PyNaCl developers | 2004 | +| PyYAML | 5.3.1 | MIT/X11 | https://pypi.org/project/PyYAML/ | Kirill Simonov | 2006-2019 | +| WerkZeug | 1.0.1 | BSD | https://pypi.org/project/Werkzeug/ | Armin Ronacher | 2007 | +| apache-libcloud | 2.0.0 | Apache License, V2.0 | https://pypi.org/project/apache-libcloud/ | Apache Software Foundation | 2004 | +| appdirs | 1.4.4 | MIT/X11 | https://pypi.org/project/appdirs/ | Trent Mick | 2010 | +| asn1crypto | 1.3.0 | MIT/X11 | https://pypi.org/project/asn1crypto/ | wbond | 2015-2019 | +| attrs | 19.3.1 | MIT/X11 | https://pypi.org/project/attrs/ | Hynek Schlawack | 2015 | +| aws-sam-translator | 1.25.0 | Apache License, V2.0 | https://pypi.org/project/aws-sam-translator/ | Amazon Web Services | 2004 | +| aws-xray-sdk | 2.6.0 | Apache License, V2.0 | https://pypi.org/project/aws-xray-sdk/ | Amazon Web Services | 2004 | +| bcrypt | 3.1.7 | Apache License, V2.0 | https://pypi.org/project/bcrypt/ | The Python Cryptographic Authority | 2013 | +| boto | 2.49.0 | MIT/X11 | https://pypi.org/project/boto/ | Mitch Garnaatt | 2013 | +| boto3 | 1.14.16 | Apache License, V2.0 | https://pypi.org/project/boto3/ | AWS | 2019 | +| botocore | 1.17.16 | Apache License, V2.0 | https://pypi.org/project/botocore/ | AWS | 2019 | +| cachetools | 4.1.1 | MIT/X11 | https://pypi.org/project/cachetools/ | tkem | 2014-2020 | +| cassandra-driver | 3.24.0 | Apache License, V2.0 | https://pypi.org/project/cassandra-driver/ | DataStax | 2013 | +| certifi | 2020.6.20 | Mozilla Public License, V2.0 | https://pypi.org/project/certifi/ | Kenneth Reitz | 2013 | +| certvalidator | 0.11.1 | MIT/X11 | https://pypi.org/project/certvalidator/ | wbond | 2015-2018 | +| cffi | 1.14.0 | MIT/X11 | https://pypi.org/project/cffi/ | Armin Rigo, Maciej Fijalkowski | 2005-2007 | +| cfn-lint | 0.33.2 | MIT/X11 | https://pypi.org/project/cfn-lint/ | kddejong | 2018 | +| chardet | 3.0.4 | GNU Lesser General Public License, V2.1 | https://pypi.org/project/chardet/ | Daniel Blanchard | 2012 | +| cheroot | 8.3.0 | BSD | https://pypi.org/project/cheroot/ | CherryPy Team | 2004-2020 | +| click | 7.1.2 | BSD | https://pypi.org/project/click/ | Pallets | 2014 | +| contextlib2 | 0.6.0.post1 | Python License | https://pypi.org/project/contextlib2/ | Nick Coghlan | 2015 | +| croniter | 0.3.34 | MIT/X11 | https://pypi.org/project/croniter/ | Matsumoto Taichi, kiorky | 2014 | +| cryptography | 2.9.2 | BSD | https://pypi.org/project/cryptography/ | The cryptography developers | 2014 | +| decorator | 4.4.2 | BSD | https://pypi.org/project/decorator/ | Michele Simionato | 2005-2018 | +| distlib | 0.3.1 | Python License | https://pypi.org/project/distlib/ | Vinay Sajip | 2012 | +| dnspython | 1.16.0 | BSD | https://pypi.org/project/dnspython/ | Bob Halley | 2001-2017 | +| docker | 4.2.2 | Apache License, V2.0 | https://pypi.org/project/docker/ | Joffery F | 2016 | +| docutils | 0.15.2 | BSD | https://pypi.org/project/docutils/ | David Goodger | 2011 | +| ecdsa | 0.15 | MIT/X11 | https://pypi.org/project/ecdsa/ | Brian Warner | 2010 | +| filelock | 3.0.12 | Public Domain | https://pypi.org/project/filelock/ | Benedikt Schmitt | 2018 | +| future | 0.18.2 | MIT/X11 | https://pypi.org/project/future/ | Ed Schofield | 2013-2019 | +| geomet | 0.2.1.post1 | Apache License, V2.0 | https://pypi.org/project/geomet/ | Lars Butler | 2014 | +| gitdb | 4.0.5 | BSD | https://pypi.org/project/gitdb/ | Sebastian Thiel | 2010, 2011 | +| google-auth | 1.18.0 | Apache License, V2.0 | https://pypi.org/project/google-auth/ | Google Cloud Platform | 2016 | +| hgtools | 8.2.0 | MIT/X11 | https://pypi.org/project/hgtools/ | Jason R. Coombs | 2019 | +| idna | 2.8 | BSD | https://pypi.org/project/idna/ | Kim Davies | 2013-2020 | +| importlib-metadata | 1.7.0 | Apache License, V2.0 | https://pypi.org/project/importlib-metadata/ | Barry Warsaw | 2017-2019 | +| ipaddress | 1.0.23 | Python License | https://pypi.org/project/ipaddress/ | Philipp Hagemeister | 2014 | +| isodate | 0.6.0 | BSD | https://pypi.org/project/isodate/ | Gerhard Weis | 2009 | +| jaraco.functools | 3.0.1 | MIT/X11 | https://pypi.org/project/jaraco.functools/ | Jason R. Coombs | 2020 | +| jmespath | 0.10.0 | MIT/X11 | https://pypi.org/project/jmespath/ | James Saryerwinnie | 2013 | +| jsondiff | 1.1.2 | MIT/X11 | https://pypi.org/project/jsondiff/ | Zoomer Analytics LLC | 2015 | +| jsonpatch | 1.26 | BSD | https://pypi.org/project/jsonpatch/ | Stefan Kögl | 2011 | +| jsonpickle | 1.4.1 | BSD | https://pypi.org/project/jsonpickle/ | David Aguilar | 2009-2018 | +| jsonpointer | 2 | BSD | https://pypi.org/project/jsonpointer/ | Stefan Kögl | 2017 | +| jsonschema | 3.2.0 | MIT/X11 | https://pypi.org/project/jsonschema/ | Julian Berman | 2013 | +| junit-xml | 1.9 | MIT/X11 | https://pypi.org/project/junit-xml/ | Brian Beyer | 2013 | +| junos-eznc | 2.4.0 | Apache License, V2.0 | https://pypi.org/project/junos-eznc/ | Jeremy Schulman, Nitin Kumar, Rick Sherman, Stacy Smith | 2018 | +| jxmlease | 1.0.3 | MIT/X11 | https://pypi.org/project/jxmlease/ | Juniper Networks | 2015-2016 | +| kazoo | 2.7.0 | Apache License, V2.0 | https://pypi.org/project/kazoo/ | Kazoo team | 2012 | +| keyring | 5.7.1 | MIT/X11 | https://pypi.org/project/keyring/ | Kang Zhang | 2019 | +| kubernetes | 3.0.0 | Apache License, V2.0 | https://pypi.org/project/kubernetes/ | Kubernetes | 2016 | +| libnacl | 1.7.1 | Apache License, V2.0 | https://pypi.org/project/libnacl/ | Thomas S Hatch | 2014 | +| lxml | 4.5.1 | BSD | https://pypi.org/project/lxml/ | lxml dev team | 2019 | +| mock | 4.0.2 | BSD | https://pypi.org/project/mock/ | Testing Cabal | 2003-2013 | +| more-itertools | 5.0.0 | MIT/X11 | https://pypi.org/project/more-itertools/ | Eric Rose | 2012 | +| moto | 1.3.14 | Apache License, V2.0 | https://pypi.org/project/moto/ | Steve Pulec | 2004 | +| natsort | 7.0.1 | MIT/X11 | https://pypi.org/project/natsort/ | Seth M. Morton | 2012-2020 | +| ncclient | 0.6.7 | Apache License, V2.0 | https://pypi.org/project/ncclient/ | Shikhar Bhushan, Leonidas Poulopoulos, Ebben Aries, Einar Nilsen-Nygaard | 2004 | +| netaddr | 0.8.0 | BSD | https://pypi.org/project/netaddr/ | Author: David P. D. Moss, Stefan Nordhausen et al | 2008 | +| networkx | 2.4 | BSD | https://pypi.org/project/networkx/ | Aric Hagberg | 2004-2020 | +| ntc-templates | 1.5.0 | Apache License, V2.0 | https://pypi.org/project/ntc-templates/ | Jason Edelman | 2015 | +| oauthlib | 3.1.0 | BSD | https://pypi.org/project/oauthlib/ | The OAuthlib Community | 2019 | +| oscrypto | 1.2.0 | MIT/X11 | https://pypi.org/project/oscrypto/ | wbond | 2015-2019 | +| paramiko | 2.7.1 | GNU Lesser General Public License, V2.1 | https://pypi.org/project/paramiko/ | Jeff Forcier | 1999 | +| passlib | 1.7.2 | BSD | https://pypi.org/project/passlib/ | Eli Collins | 2004 | +| pathtools | 0.1.2 | MIT/X11 | https://pypi.org/project/pathtools/ | Yesudeep Mangalapilly | 2010 | +| portend | 2.6 | MIT/X11 | https://pypi.org/project/portend/ | Jason R Coombs | 2018 | +| psutil | 5.7.0 | BSD | https://pypi.org/project/psutil/ | Giampaolo Rodola | 2009 | +| pyOpenSSL | 19.1.0 | Apache License, V2.0 | https://pypi.org/project/pyOpenSSL/ | The pyopenssl Developers | 2004 | +| pyasn1 | 0.4.8 | BSD | https://pypi.org/project/pyasn1/ | Ilya Etingof | 2005-2020 | +| pyasn1-modules | 0.2.8 | BSD | https://pypi.org/project/pyasn1-modules/ | Ilya Etingof | 2005-2020 | +| pycparser | 2.20 | BSD | https://pypi.org/project/pycparser/ | Eli Bendersky | 2008-2017 | +| pygit2 | 1.2.1 | GNU General Public License, V2.0 | https://pypi.org/project/pygit2/ | J. David Ibáñez | 2012 | +| pyinotify | 0.9.6 | MIT/X11 | https://pypi.org/project/pyinotify/ | Sebastien Martini | 2010 | +| pyparsing | 2.4.7 | MIT/X11 | https://pypi.org/project/pyparsing/ | Paul McGuire | 2018 | +| pyrsistent | 0.16.0 | MIT/X11 | https://pypi.org/project/pyrsistent/ | Tobias Gustafsson | 2019 | +| pyserial | 3.4 | BSD | https://pypi.org/project/pyserial/ | Chris Liechti | 2001-2016 | +| python-dateutil | 2.8.1 | Apache License, V2.0 | https://pypi.org/project/python-dateutil/ | Gustavo Niemeyer | 2017 | +| python-etcd | 0.4.5 | MIT/X11 | https://pypi.org/project/python-etcd/ | Jose Plana | 2015 | +| python-gnupg | 0.4.6 | BSD | https://pypi.org/project/python-gnupg/ | Vinay Sajip | 2008-2019 | +| python-jose[cryptography] | 3.2.0 | MIT/X11 | https://pypi.org/project/python-jose/ | Michael Davis | 2015 | +| pytz | 2020.1 | MIT/X11 | https://pypi.org/project/pytz/ | Stuart Bishop | 2015 | +| pyvmomi | 7 | Apache License, V2.0 | https://pypi.org/project/pyvmomi/ | VMware, Inc. | 2004 | +| requests | 2.24.0 | Apache License, V2.0 | https://pypi.org/project/requests/ | Kenneth Reitz | 2004 | +| requests-oauthlib | 1.3.0 | BSD | https://pypi.org/project/requests-oauthlib/ | Kenneth Reitz | 2014 | +| responses | 0.10.15 | Apache License, V2.0 | https://pypi.org/project/responses/ | David Cramer | 2004 | +| rfc3987 | 1.3.8 | GNU General Public License, V3.0 | https://pypi.org/project/rfc3987/ | Daniel Gerber | 2007 | +| rsa | 4.6 | Apache License, V2.0 | https://pypi.org/project/rsa/ | Sybren A. Stuvel | 2011 | +| s3transfer | 0.3.3 | Apache License, V2.0 | https://pypi.org/project/s3transfer/ | Amazon Web Services | 2004 | +| scp | 0.13.2 | GNU Lesser General Public License, V2.1 | https://pypi.org/project/scp/ | James Bardin | 2013 | +| setproctitle | 1.1.10 | BSD | https://pypi.org/project/setproctitle/ | Daniele Varrazzo | 2009-2020 | +| setuptools-scm | 4.1.2 | MIT/X11 | https://pypi.org/project/setuptools-scm/ | Ronny Pfannschmidt | 2010 | +| six | 1.15.0 | MIT/X11 | https://pypi.org/project/six/ | Benjamin Peterson | 2010-2020 | +| smmap | 3.0.4 | BSD | https://pypi.org/project/smmap/ | Sebastian Thiel | 2010, 2011 | +| sshpubkeys | 3.1.0 | BSD | https://pypi.org/project/sshpubkeys/ | Olli Jarva | 2014 | +| strict-rfc3339 | 0.7 | GNU General Public License, V3.0 | https://pypi.org/project/strict-rfc3339/ | Daniel Richman, Adam Greig | 2007 | +| tempora | 3.0.0 | MIT/X11 | https://pypi.org/project/tempora/ | Jason R. Coombs | 2020 | +| textfsm | 1.1.0 | Apache License, V2.0 | https://pypi.org/project/textfsm/ | Google | 2004 | +| timelib | 0.2.4 | BSD | https://pypi.org/project/timelib/ | Ralf Schmitt | 2009-2011 | +| toml | 0.10.1 | MIT/X11 | https://pypi.org/project/toml/ | William Pearson | 2013-2019 | +| transitions | 0.8.2 | MIT/X11 | https://pypi.org/project/transitions/ | Tal Yarkoni | 2014 - 2020 | +| urllib3 | 1.25.9 | MIT/X11 | https://pypi.org/project/urllib3/ | Andrey Petrov | 2008-2020 | +| vcert | 0.7.4 | ASL | https://pypi.org/project/vcert | Denis Subbotin | 2018 | +| virtualenv | 20.0.25 | MIT | https://pypi.org/project/virtualenv | Bernat Gabor | 2020-202x | +| watchdog | 0.10.3 | Apache License 2.0 | https://pypi.org/project/watchdog | Yesudeep Mangalapilly | 2012 | +| websocket-client | 0.57.0 | BSD | https://pypi.org/project/websocket-client | liris | 2018 | +| wrapt | 1.12.1 | BSD | https://pypi.org/project/wrapt | Graham Dumpleton | 2013-2019 | +| xmltodict | 0.12.0 | MIT | https://pypi.org/project/xmltodict | Martin Blech | 2010 | +| yamlordereddictloader | 0.4.0 | MIT License | https://pypi.org/project/yamlordereddictloader | François Ménabé | 2017 | +| zc.lockfile | 2 | ZPL 2.1 | https://pypi.org/project/zc.lockfile | Zope Foundation | 2010 | +| zipp | 3.1.0 | MIT | https://pypi.org/project/zipp | Jason R. Coombs | 2019 | diff --git a/doc/conf.py b/doc/conf.py index 653d912c20d..f85dfef159b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -321,7 +321,6 @@ linkcheck_ignore = [ r"https://salt-cloud.readthedocs.io", r"https://salt.readthedocs.io", r"http://www.pip-installer.org/", - r"http://www.windowsazure.com/", r"https://github.com/watching", r"dash-feed://", r"https://github.com/saltstack/salt/", diff --git a/doc/ref/clouds/all/index.rst b/doc/ref/clouds/all/index.rst index 397e7f77ffa..14829dec454 100644 --- a/doc/ref/clouds/all/index.rst +++ b/doc/ref/clouds/all/index.rst @@ -11,7 +11,6 @@ cloud modules :template: autosummary.rst.tmpl aliyun - azurearm clc cloudstack digitalocean @@ -24,7 +23,6 @@ cloud modules libvirt linode lxc - msazure oneandone opennebula openstack diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 74d4b58b084..bd1ccdde2e1 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -4046,29 +4046,6 @@ This option defines the update interval (in seconds) for :ref:`MinionFS minionfs_update_interval: 120 -azurefs: Azure File Server Backend ----------------------------------- - -.. versionadded:: 2015.8.0 - -See the :mod:`azurefs documentation ` for usage -examples. - -.. conf_master:: azurefs_update_interval - -``azurefs_update_interval`` -*************************** - -.. versionadded:: 2018.3.0 - -Default: ``60`` - -This option defines the update interval (in seconds) for azurefs. - -.. code-block:: yaml - - azurefs_update_interval: 120 - s3fs: S3 File Server Backend ---------------------------- diff --git a/doc/ref/file_server/all/index.rst b/doc/ref/file_server/all/index.rst index c6c13e291bc..ce06ea14a06 100644 --- a/doc/ref/file_server/all/index.rst +++ b/doc/ref/file_server/all/index.rst @@ -10,7 +10,6 @@ fileserver modules :toctree: :template: autosummary.rst.tmpl - azurefs gitfs hgfs minionfs diff --git a/doc/ref/grains/all/index.rst b/doc/ref/grains/all/index.rst index 6fd9ae05c9f..80ee46f7189 100644 --- a/doc/ref/grains/all/index.rst +++ b/doc/ref/grains/all/index.rst @@ -25,7 +25,6 @@ grains modules mdadm mdata metadata - metadata_azure metadata_gce minion_process napalm diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index cbd8b0cdc52..e5223cdf58b 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -42,10 +42,6 @@ execution modules at_solaris augeas_cfg aws_sqs - azurearm_compute - azurearm_dns - azurearm_network - azurearm_resource bamboohr baredoc bcache diff --git a/doc/ref/pillar/all/index.rst b/doc/ref/pillar/all/index.rst index b06cf2f6859..b5e60f9a1d8 100644 --- a/doc/ref/pillar/all/index.rst +++ b/doc/ref/pillar/all/index.rst @@ -10,7 +10,6 @@ pillar modules :toctree: :template: autosummary.rst.tmpl - azureblob cmd_json cmd_yaml cmd_yamlex diff --git a/doc/ref/states/all/index.rst b/doc/ref/states/all/index.rst index 13ff645b59f..00355c022ff 100644 --- a/doc/ref/states/all/index.rst +++ b/doc/ref/states/all/index.rst @@ -24,10 +24,6 @@ state modules at augeas aws_sqs - azurearm_compute - azurearm_dns - azurearm_network - azurearm_resource beacon bigip blockdev diff --git a/doc/spelling_wordlist.txt b/doc/spelling_wordlist.txt index 265dfaca0a0..ac88f21c058 100644 --- a/doc/spelling_wordlist.txt +++ b/doc/spelling_wordlist.txt @@ -426,7 +426,6 @@ mongodb monit moosefs mountpoint -msazure msc msdos msg diff --git a/doc/topics/cloud/cloud.rst b/doc/topics/cloud/cloud.rst index bfb7aff6bdc..6df4a30c76b 100644 --- a/doc/topics/cloud/cloud.rst +++ b/doc/topics/cloud/cloud.rst @@ -28,10 +28,7 @@ upon execution. Most often, it uses ``get_configured_provider()`` to determine if the necessary configuration has been set up. It may also check for necessary imports, to decide whether to load the module. In most cases, it will return a ``True`` or ``False`` value. If the name of the driver used does not match the -filename, then that name should be returned instead of ``True``. An example of -this may be seen in the Azure module: - -https://github.com/saltstack/salt/tree/|repo_primary_branch|/salt/cloud/clouds/msazure.py +filename, then that name should be returned instead of ``True``. The get_configured_provider() Function -------------------------------------- @@ -240,8 +237,7 @@ The script() Function --------------------- This function builds the deploy script to be used on the remote machine. It is likely to be moved into the ``salt.utils.cloud`` library in the near future, as -it is very generic and can usually be copied wholesale from another module. An -excellent example is in the Azure driver. +it is very generic and can usually be copied wholesale from another module. The destroy() Function ---------------------- diff --git a/doc/topics/cloud/index.rst b/doc/topics/cloud/index.rst index ce0bd82232a..c279ed5f256 100644 --- a/doc/topics/cloud/index.rst +++ b/doc/topics/cloud/index.rst @@ -106,8 +106,6 @@ Cloud Provider Specifics :maxdepth: 3 Getting Started With Aliyun - Getting Started With Azure - Getting Started With Azure Arm Getting Started With CloudStack Getting Started With DigitalOcean Getting Started With Dimension Data diff --git a/requirements/static/ci/common.in b/requirements/static/ci/common.in index addf9a7bafd..a6700e13084 100644 --- a/requirements/static/ci/common.in +++ b/requirements/static/ci/common.in @@ -1,7 +1,6 @@ # Requirements in this file apply to all platforms. # We can also exclude platforms from the requirements using markers, but if a requirement only applies # to a particular platform, please add it to the corresponding `.in` file in this directory. -azure==4.0.0; sys_platform != 'win32' apache-libcloud>=1.5.0; sys_platform != 'win32' boto3>=1.16.0,<1.17.0; python_version < '3.6' boto3>=1.17.67; python_version >= '3.6' diff --git a/requirements/static/ci/py3.10/cloud.txt b/requirements/static/ci/py3.10/cloud.txt index 4efb5c1129b..669ec1f31f4 100644 --- a/requirements/static/ci/py3.10/cloud.txt +++ b/requirements/static/ci/py3.10/cloud.txt @@ -4,10 +4,6 @@ # # pip-compile --output-file=requirements/static/ci/py3.10/cloud.txt --pip-args='--constraint=requirements/static/ci/py3.10/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/cloud.in requirements/static/ci/common.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.7 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -31,304 +27,6 @@ attrs==21.2.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.27 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.52 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.2.0 @@ -354,14 +52,12 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # requests certvalidator==0.11.1 # via vcert cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # pynacl @@ -388,10 +84,6 @@ croniter==1.0.15 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -449,8 +141,6 @@ iniconfig==1.1.1 # via pytest ipaddress==1.0.23 # via kubernetes -isodate==0.6.0 - # via msrest jaraco.classes==3.2.1 # via jaraco.collections jaraco.collections==3.4.0 @@ -479,14 +169,10 @@ junos-eznc==2.6.0 ; sys_platform != "win32" and python_version <= "3.10" # via -r requirements/static/ci/common.in jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -516,98 +202,6 @@ msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.21 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -620,17 +214,14 @@ netaddr==0.7.19 # junos-eznc ntlm-auth==1.3.0 # via requests-ntlm -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.1 # via certvalidator packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # ncclient # scp @@ -663,10 +254,6 @@ pycparser==2.21 ; python_version >= "3.9" # cffi pycryptodomex==3.10.1 # via -r requirements/crypto.txt -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in -pyjwt==2.4.0 - # via adal pynacl==1.4.0 # via paramiko pyopenssl==23.0.0 @@ -730,9 +317,6 @@ pytest==7.2.0 ; python_version > "3.6" python-dateutil==2.8.2 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -763,30 +347,19 @@ pyzmq==23.2.0 ; python_version < "3.11" # pytest-salt-factories requests-ntlm==1.1.0 # via pywinrm -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # profitbricks # pyvmomi # pywinrm # requests-ntlm - # requests-oauthlib # responses # vcert responses==0.14.0 @@ -813,12 +386,9 @@ six==1.16.0 # etcd3-py # genshi # geomet - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes - # msrestazure # ncclient # paramiko # profitbricks diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index d6789f835f7..7118f507b19 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -4,10 +4,6 @@ # # pip-compile --output-file=requirements/static/ci/py3.10/darwin.txt --pip-args='--constraint=requirements/static/pkg/py3.10/darwin.txt' requirements/darwin.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/darwin.in requirements/static/pkg/darwin.in # -adal==1.2.5 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -31,304 +27,6 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.26 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.51 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 @@ -354,14 +52,12 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # requests certvalidator==0.11.1 # via vcert cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # pygit2 @@ -389,10 +85,6 @@ croniter==0.3.29 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/darwin.txt - # adal - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -452,8 +144,6 @@ iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest jaraco.classes==3.2.1 # via jaraco.collections jaraco.collections==3.4.0 @@ -519,98 +209,6 @@ msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.19 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -619,8 +217,6 @@ ncclient==0.6.9 # via junos-eznc netaddr==0.7.19 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -666,8 +262,6 @@ pycryptodomex==3.9.8 # via -r requirements/crypto.txt pygit2==1.9.1 ; python_version >= "3.7" # via -r requirements/static/ci/darwin.in -pyjwt==2.4.0 - # via adal pynacl==1.3.0 # via paramiko pyopenssl==23.0.0 @@ -725,9 +319,6 @@ pytest==7.2.0 ; python_version > "3.6" python-dateutil==2.8.0 # via # -r requirements/darwin.txt - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -755,27 +346,16 @@ pyzmq==23.2.0 ; python_version < "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # pyvmomi - # requests-oauthlib # responses # vcert # vultr @@ -802,12 +382,10 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kubernetes # mock - # msrestazure # ncclient # paramiko # pynacl diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index f7c63cdfa59..bf080132823 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -4,10 +4,6 @@ # # pip-compile --output-file=requirements/static/ci/py3.10/freebsd.txt --pip-args='--constraint=requirements/static/pkg/py3.10/freebsd.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/freebsd.in requirements/static/pkg/freebsd.in requirements/zeromq.txt # -adal==1.2.5 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -29,304 +25,6 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.26 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.51 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 @@ -359,7 +57,6 @@ certvalidator==0.11.1 cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # pygit2 @@ -386,10 +83,6 @@ croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in cryptography==39.0.2 # via - # adal - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -516,98 +209,6 @@ msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.19 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -724,9 +325,6 @@ pytest==7.2.0 ; python_version > "3.6" python-dateutil==2.8.1 # via # -r requirements/static/pkg/freebsd.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -760,14 +358,7 @@ requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes @@ -807,7 +398,6 @@ six==1.16.0 # kubernetes # mock # more-itertools - # msrestazure # ncclient # paramiko # pynacl diff --git a/requirements/static/ci/py3.10/lint.txt b/requirements/static/ci/py3.10/lint.txt index 5ca4fd978cb..d5936509f46 100644 --- a/requirements/static/ci/py3.10/lint.txt +++ b/requirements/static/ci/py3.10/lint.txt @@ -4,10 +4,6 @@ # # pip-compile --output-file=requirements/static/ci/py3.10/lint.txt --pip-args='--constraint=requirements/static/ci/py3.10/linux.txt' requirements/base.txt requirements/static/ci/common.in requirements/static/ci/lint.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.7 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -32,304 +28,6 @@ attrs==21.2.0 # via # aiohttp # jsonschema -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.27 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.52 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.2.0 @@ -365,7 +63,6 @@ certvalidator==0.11.1 cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # pygit2 @@ -393,11 +90,7 @@ croniter==1.0.15 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal # ansible-core - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -518,98 +211,6 @@ moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 # via -r requirements/base.txt -msrest==0.6.21 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -692,9 +293,6 @@ python-consul==1.1.0 python-dateutil==2.8.2 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -738,12 +336,6 @@ requests==2.26.0 # -r requirements/static/ci/common.in # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes @@ -790,7 +382,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # msrestazure # ncclient # paramiko # pynacl diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index 271c1edd344..040450adf01 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -4,10 +4,6 @@ # # pip-compile --output-file=requirements/static/ci/py3.10/linux.txt --pip-args='--constraint=requirements/static/pkg/py3.10/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.3 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -35,309 +31,6 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.18 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.5 - # via azure -azure-datalake-store==0.0.44 - # via azure -azure-eventgrid==1.2.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.0 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.4.1 - # via azure-mgmt -azure-mgmt-containerregistry==2.7.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.5.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.0 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.6.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.0.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.8.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.1.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.0.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.6 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.0 - # via - # azure-cosmosdb-table - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 @@ -373,7 +66,6 @@ certvalidator==0.11.1 cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # pygit2 @@ -401,11 +93,7 @@ croniter==0.3.29 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal # ansible-core - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -532,95 +220,6 @@ msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.14 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.3 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -744,9 +343,6 @@ python-consul==1.1.0 python-dateutil==2.8.1 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -791,14 +387,7 @@ requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes diff --git a/requirements/static/ci/py3.11/darwin-crypto.txt b/requirements/static/ci/py3.11/darwin-crypto.txt new file mode 100644 index 00000000000..32d8f607198 --- /dev/null +++ b/requirements/static/ci/py3.11/darwin-crypto.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/static/ci/py3.11/darwin-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.11/darwin.txt' requirements/static/ci/crypto.in +# +m2crypto==0.38.0 + # via -r requirements/static/ci/crypto.in +pycryptodome==3.18.0 + # via -r requirements/static/ci/crypto.in diff --git a/requirements/static/ci/py3.11/darwin.txt b/requirements/static/ci/py3.11/darwin.txt index d9c608388db..a74d24b212e 100644 --- a/requirements/static/ci/py3.11/darwin.txt +++ b/requirements/static/ci/py3.11/darwin.txt @@ -4,10 +4,6 @@ # # pip-compile --output-file=requirements/static/ci/py3.11/darwin.txt --pip-args='--constraint=requirements/static/pkg/py3.11/darwin.txt' requirements/darwin.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/darwin.in requirements/static/pkg/darwin.in # -adal==1.2.5 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -31,304 +27,6 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.26 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.51 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 @@ -352,14 +50,12 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # requests certvalidator==0.11.1 # via vcert cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # pygit2 @@ -386,10 +82,6 @@ croniter==0.3.29 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/darwin.txt - # adal - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # pyopenssl @@ -446,8 +138,6 @@ iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest jaraco.classes==3.2.1 # via jaraco.collections jaraco.collections==3.4.0 @@ -506,104 +196,10 @@ msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.19 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp # yarl -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -644,8 +240,6 @@ pycryptodomex==3.9.8 # via -r requirements/crypto.txt pygit2==1.9.1 ; python_version >= "3.7" # via -r requirements/static/ci/darwin.in -pyjwt==2.4.0 - # via adal pyopenssl==23.0.0 # via # -r requirements/darwin.txt @@ -697,9 +291,6 @@ pytest==7.2.0 ; python_version > "3.6" python-dateutil==2.8.0 # via # -r requirements/darwin.txt - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -726,27 +317,16 @@ pyzmq==25.0.2 ; python_version >= "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # pyvmomi - # requests-oauthlib # responses # vcert # vultr @@ -771,11 +351,9 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # kubernetes # mock - # msrestazure # python-dateutil # pyvmomi # responses diff --git a/requirements/static/ci/py3.11/freebsd-crypto.txt b/requirements/static/ci/py3.11/freebsd-crypto.txt new file mode 100644 index 00000000000..535a2529e8f --- /dev/null +++ b/requirements/static/ci/py3.11/freebsd-crypto.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/static/ci/py3.11/freebsd-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.11/freebsd.txt' requirements/static/ci/crypto.in +# +m2crypto==0.38.0 + # via -r requirements/static/ci/crypto.in +pycryptodome==3.18.0 + # via -r requirements/static/ci/crypto.in diff --git a/requirements/static/ci/py3.11/freebsd.txt b/requirements/static/ci/py3.11/freebsd.txt index 0e45bcde4ad..ae52c65407e 100644 --- a/requirements/static/ci/py3.11/freebsd.txt +++ b/requirements/static/ci/py3.11/freebsd.txt @@ -4,10 +4,6 @@ # # pip-compile --output-file=requirements/static/ci/py3.11/freebsd.txt --pip-args='--constraint=requirements/static/pkg/py3.11/freebsd.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/freebsd.in requirements/static/pkg/freebsd.in requirements/zeromq.txt # -adal==1.2.5 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -29,304 +25,6 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.26 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.51 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 @@ -359,7 +57,6 @@ certvalidator==0.11.1 cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # pygit2 @@ -386,10 +83,6 @@ croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in cryptography==39.0.2 # via - # adal - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -507,98 +200,6 @@ msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.19 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -703,9 +304,6 @@ pytest==7.2.0 ; python_version > "3.6" python-dateutil==2.8.1 # via # -r requirements/static/pkg/freebsd.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -737,14 +335,7 @@ requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes @@ -781,7 +372,6 @@ six==1.16.0 # kubernetes # mock # more-itertools - # msrestazure # paramiko # pynacl # python-dateutil diff --git a/requirements/static/ci/py3.11/linux-crypto.txt b/requirements/static/ci/py3.11/linux-crypto.txt new file mode 100644 index 00000000000..69646264e97 --- /dev/null +++ b/requirements/static/ci/py3.11/linux-crypto.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/static/ci/py3.11/linux-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.11/linux.txt' requirements/static/ci/crypto.in +# +m2crypto==0.38.0 + # via -r requirements/static/ci/crypto.in +pycryptodome==3.18.0 + # via -r requirements/static/ci/crypto.in diff --git a/requirements/static/ci/py3.11/linux.txt b/requirements/static/ci/py3.11/linux.txt index 2639d371cc4..869451c0f6f 100644 --- a/requirements/static/ci/py3.11/linux.txt +++ b/requirements/static/ci/py3.11/linux.txt @@ -4,10 +4,6 @@ # # pip-compile --output-file=requirements/static/ci/py3.11/linux.txt --pip-args='--constraint=requirements/static/pkg/py3.11/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.3 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -35,309 +31,6 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.18 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.5 - # via azure -azure-datalake-store==0.0.44 - # via azure -azure-eventgrid==1.2.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.0 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.4.1 - # via azure-mgmt -azure-mgmt-containerregistry==2.7.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.5.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.0 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.6.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.0.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.8.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.1.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.0.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.6 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.0 - # via - # azure-cosmosdb-table - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 @@ -373,7 +66,6 @@ certvalidator==0.11.1 cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # pygit2 @@ -403,9 +95,6 @@ cryptography==39.0.2 # -r requirements/static/pkg/linux.in # adal # ansible-core - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -523,95 +212,6 @@ msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.14 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.3 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -723,9 +323,6 @@ python-consul==1.1.0 python-dateutil==2.8.1 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -768,14 +365,7 @@ requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes diff --git a/requirements/static/ci/py3.11/windows-crypto.txt b/requirements/static/ci/py3.11/windows-crypto.txt new file mode 100644 index 00000000000..14e0be2eeee --- /dev/null +++ b/requirements/static/ci/py3.11/windows-crypto.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/static/ci/py3.11/windows-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.11/windows.txt' requirements/static/ci/crypto.in +# +m2crypto==0.38.0 + # via -r requirements/static/ci/crypto.in +pycryptodome==3.18.0 + # via -r requirements/static/ci/crypto.in diff --git a/requirements/static/ci/py3.7/cloud.txt b/requirements/static/ci/py3.7/cloud.txt index 857a9d054bc..2aefd51e45e 100644 --- a/requirements/static/ci/py3.7/cloud.txt +++ b/requirements/static/ci/py3.7/cloud.txt @@ -4,10 +4,6 @@ # # pip-compile --output-file=requirements/static/ci/py3.7/cloud.txt --pip-args='--constraint=requirements/static/ci/py3.7/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/cloud.in requirements/static/ci/common.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.7 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -33,304 +29,6 @@ attrs==21.2.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.27 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.52 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.2.0 @@ -356,14 +54,12 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # requests certvalidator==0.11.1 # via vcert cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -395,10 +91,6 @@ croniter==1.0.15 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -470,8 +162,6 @@ iniconfig==1.1.1 # via pytest ipaddress==1.0.23 # via kubernetes -isodate==0.6.0 - # via msrest jaraco.classes==3.2.1 # via jaraco.collections jaraco.collections==3.4.0 @@ -503,14 +193,10 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in loguru==0.6.0 # via ciscoconfparse looseversion==1.0.2 @@ -543,98 +229,6 @@ msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.21 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -659,17 +253,14 @@ ntc-templates==2.3.2 # netmiko ntlm-auth==1.3.0 # via requests-ntlm -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.1 # via certvalidator packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -705,10 +296,6 @@ pycryptodomex==3.10.1 # via -r requirements/crypto.txt pyeapi==0.8.4 # via napalm -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in -pyjwt==2.4.0 - # via adal pynacl==1.4.0 # via paramiko pyopenssl==23.0.0 @@ -774,9 +361,6 @@ pytest==7.2.0 ; python_version > "3.6" python-dateutil==2.8.2 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -808,31 +392,20 @@ pyzmq==23.2.0 ; python_version < "3.11" # pytest-salt-factories requests-ntlm==1.1.0 # via pywinrm -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # profitbricks # pyvmomi # pywinrm # requests-ntlm - # requests-oauthlib # responses # vcert responses==0.14.0 @@ -862,12 +435,9 @@ six==1.16.0 # etcd3-py # genshi # geomet - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes - # msrestazure # ncclient # paramiko # profitbricks diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt index 71f214e61b6..778cdda0242 100644 --- a/requirements/static/ci/py3.7/freebsd.txt +++ b/requirements/static/ci/py3.7/freebsd.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/static/ci/py3.7/freebsd.txt --pip-args='--constraint=requirements/static/pkg/py3.7/freebsd.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/freebsd.in requirements/static/pkg/freebsd.in requirements/zeromq.txt # -adal==1.2.5 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -20,8 +16,6 @@ asn1crypto==1.3.0 # oscrypto async-timeout==4.0.2 # via aiohttp -asynctest==0.13.0 - # via aiohttp attrs==20.3.0 # via # aiohttp @@ -31,316 +25,18 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.26 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.51 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in -backports.entry-points-selectable==1.1.0 +backports-entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 # via # paramiko # passlib +boto==2.49.0 + # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto -boto==2.49.0 - # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -354,14 +50,12 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # requests certvalidator==0.11.1 # via vcert cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -393,10 +87,6 @@ croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in cryptography==39.0.2 # via - # adal - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -454,32 +144,22 @@ idna==2.8 immutables==0.15 # via contextvars importlib-metadata==4.6.4 - # via - # -r requirements/static/pkg/freebsd.in - # backports.entry-points-selectable - # jsonschema - # mako - # moto - # pluggy - # pytest - # virtualenv + # via -r requirements/static/pkg/freebsd.in iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest -jaraco.classes==3.2.1 - # via jaraco.collections -jaraco.collections==3.4.0 +jaraco-classes==3.2.1 + # via jaraco-collections +jaraco-collections==3.4.0 # via cherrypy -jaraco.functools==2.0 +jaraco-functools==2.0 # via # cheroot - # jaraco.text + # jaraco-text # tempora -jaraco.text==3.5.1 - # via jaraco.collections +jaraco-text==3.5.1 + # via jaraco-collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -500,14 +180,10 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -532,106 +208,14 @@ more-itertools==5.0.0 # via # cheroot # cherrypy - # jaraco.classes - # jaraco.functools + # jaraco-classes + # jaraco-functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.19 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -649,17 +233,14 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.0 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -685,24 +266,23 @@ psutil==5.8.0 # pytest-salt-factories # pytest-shell-utilities # pytest-system-statistics -pyasn1-modules==0.2.4 - # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pycparser==2.17 - # via cffi +pyasn1-modules==0.2.4 + # via google-auth +pycparser==2.21 ; python_version >= "3.9" + # via + # -r requirements/static/ci/common.in + # -r requirements/static/pkg/freebsd.in + # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt pyeapi==0.8.3 # via napalm pygit2==1.8.0 ; python_version >= "3.7" # via -r requirements/static/ci/freebsd.in -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in -pyjwt==2.4.0 - # via adal pynacl==1.3.0 # via paramiko pyopenssl==23.0.0 @@ -719,6 +299,18 @@ pyserial==3.4 # via # junos-eznc # netmiko +pytest==7.2.0 ; python_version > "3.6" + # via + # -r requirements/pytest.txt + # pytest-custom-exit-code + # pytest-helpers-namespace + # pytest-salt-factories + # pytest-shell-utilities + # pytest-skip-markers + # pytest-subtests + # pytest-system-statistics + # pytest-tempdir + # pytest-timeout pytest-custom-exit-code==0.3.0 # via -r requirements/pytest.txt pytest-helpers-namespace==2021.4.29 @@ -747,24 +339,9 @@ pytest-tempdir==2019.10.12 # pytest-salt-factories pytest-timeout==1.4.2 # via -r requirements/pytest.txt -pytest==7.2.0 ; python_version > "3.6" - # via - # -r requirements/pytest.txt - # pytest-custom-exit-code - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-subtests - # pytest-system-statistics - # pytest-tempdir - # pytest-timeout python-dateutil==2.8.1 # via # -r requirements/static/pkg/freebsd.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -793,28 +370,17 @@ pyzmq==23.2.0 ; python_version < "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # pyvmomi - # requests-oauthlib # responses # vcert responses==0.10.6 @@ -843,14 +409,11 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes # mock # more-itertools - # msrestazure # ncclient # paramiko # pynacl @@ -887,13 +450,8 @@ transitions==0.8.1 # via junos-eznc typing-extensions==3.10.0.0 # via - # aiohttp - # async-timeout - # gitpython - # importlib-metadata # pytest-shell-utilities # pytest-system-statistics - # yarl urllib3==1.26.6 # via # botocore @@ -924,7 +482,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==1.4 +zc-lockfile==1.4 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.7/lint.txt b/requirements/static/ci/py3.7/lint.txt index f9dba2dd5ec..26c60c2b21a 100644 --- a/requirements/static/ci/py3.7/lint.txt +++ b/requirements/static/ci/py3.7/lint.txt @@ -1,21 +1,17 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/static/ci/py3.7/lint.txt --pip-args='--constraint=requirements/static/ci/py3.7/linux.txt' requirements/base.txt requirements/static/ci/common.in requirements/static/ci/lint.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.7 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 # via aiohttp -ansible-core==2.11.4 - # via ansible -ansible==4.4.0 ; python_version < "3.9" +ansible==7.5.0 ; python_version >= "3.9" # via -r requirements/static/ci/linux.in +ansible-core==2.14.6 + # via ansible apache-libcloud==3.3.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in apscheduler==3.6.3 @@ -28,324 +24,22 @@ astroid==2.3.3 # via pylint async-timeout==4.0.2 # via aiohttp -asynctest==0.13.0 - # via aiohttp attrs==21.2.0 # via # aiohttp # jsonschema -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.27 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.52 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in -backports.entry-points-selectable==1.1.0 +backports-entry-points-selectable==1.1.0 # via virtualenv -backports.zoneinfo==0.2.1 - # via tzlocal bcrypt==3.2.0 # via # paramiko # passlib +boto==2.49.0 + # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto -boto==2.49.0 - # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -361,7 +55,6 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # python-telegram-bot # requests certvalidator==0.11.1 @@ -369,7 +62,6 @@ certvalidator==0.11.1 cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -402,11 +94,7 @@ croniter==1.0.15 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal # ansible-core - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -455,31 +143,22 @@ idna==3.2 immutables==0.16 # via contextvars importlib-metadata==4.6.4 - # via - # -r requirements/static/pkg/linux.in - # backports.entry-points-selectable - # click - # jsonschema - # mako - # moto - # virtualenv + # via -r requirements/static/pkg/linux.in ipaddress==1.0.23 # via kubernetes -isodate==0.6.0 - # via msrest isort==4.3.21 # via pylint -jaraco.classes==3.2.1 - # via jaraco.collections -jaraco.collections==3.4.0 +jaraco-classes==3.2.1 + # via jaraco-collections +jaraco-collections==3.4.0 # via cherrypy -jaraco.functools==3.3.0 +jaraco-functools==3.3.0 # via # cheroot - # jaraco.text + # jaraco-text # tempora -jaraco.text==3.5.1 - # via jaraco.collections +jaraco-text==3.5.1 + # via jaraco-collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -501,16 +180,12 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in lazy-object-proxy==1.4.3 # via astroid -libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in loguru==0.6.0 # via ciscoconfparse looseversion==1.0.2 @@ -539,104 +214,12 @@ more-itertools==8.8.0 # via # cheroot # cherrypy - # jaraco.classes - # jaraco.functools + # jaraco-classes + # jaraco-functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 # via -r requirements/base.txt -msrest==0.6.21 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -658,17 +241,14 @@ ntc-templates==2.2.2 # via # junos-eznc # netmiko -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.1 # via certvalidator packaging==21.3 # via # -r requirements/base.txt # ansible-core -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -686,30 +266,29 @@ portend==2.7.1 # via cherrypy psutil==5.8.0 # via -r requirements/base.txt -pyasn1-modules==0.2.8 - # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa +pyasn1-modules==0.2.8 + # via google-auth pycodestyle==2.5.0 # via saltpylint -pycparser==2.20 - # via cffi +pycparser==2.21 ; python_version >= "3.9" + # via + # -r requirements/static/ci/common.in + # -r requirements/static/pkg/linux.in + # cffi pycryptodomex==3.10.1 # via -r requirements/crypto.txt pyeapi==0.8.4 # via napalm -pygit2==1.0.3 ; python_version <= "3.8" +pygit2==1.12.1 ; python_version > "3.8" # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # adal - # twilio + # via twilio pylint==2.4.4 # via # -r requirements/static/ci/lint.in @@ -737,9 +316,6 @@ python-consul==1.1.0 python-dateutil==2.8.2 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -772,33 +348,22 @@ pyyaml==5.4.1 # yamlordereddictloader pyzmq==23.2.0 ; python_version < "3.11" # via -r requirements/zeromq.txt -redis-py-cluster==2.1.3 - # via -r requirements/static/ci/linux.in redis==3.5.3 # via redis-py-cluster -requests-oauthlib==1.3.0 - # via msrest +redis-py-cluster==2.1.3 + # via -r requirements/static/ci/linux.in requests==2.26.0 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -835,12 +400,9 @@ six==1.16.0 # etcd3-py # genshi # geomet - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes - # msrestazure # ncclient # paramiko # pynacl @@ -883,16 +445,6 @@ transitions==0.8.8 # via junos-eznc twilio==7.9.2 # via -r requirements/static/ci/linux.in -typed-ast==1.4.1 - # via astroid -typing-extensions==3.10.0.0 - # via - # aiohttp - # async-timeout - # gitpython - # immutables - # importlib-metadata - # yarl tzlocal==3.0 # via apscheduler urllib3==1.26.6 @@ -924,7 +476,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==2.0 +zc-lockfile==2.0 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.7/linux.txt b/requirements/static/ci/py3.7/linux.txt index 9c7a7c86dc3..3730da11095 100644 --- a/requirements/static/ci/py3.7/linux.txt +++ b/requirements/static/ci/py3.7/linux.txt @@ -1,21 +1,17 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/static/ci/py3.7/linux.txt --pip-args='--constraint=requirements/static/pkg/py3.7/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.3 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 # via aiohttp -ansible-core==2.11.7 - # via ansible -ansible==4.4.0 ; python_version < "3.9" +ansible==7.5.0 ; python_version >= "3.9" # via -r requirements/static/ci/linux.in +ansible-core==2.14.6 + # via ansible apache-libcloud==2.5.0 ; sys_platform != "win32" # via -r requirements/static/ci/common.in apscheduler==3.6.3 @@ -26,8 +22,6 @@ asn1crypto==1.3.0 # oscrypto async-timeout==4.0.2 # via aiohttp -asynctest==0.13.0 - # via aiohttp attrs==20.3.0 # via # aiohttp @@ -37,321 +31,18 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.18 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.5 - # via azure -azure-datalake-store==0.0.44 - # via azure -azure-eventgrid==1.2.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.0 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.4.1 - # via azure-mgmt -azure-mgmt-containerregistry==2.7.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.5.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.0 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.6.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.0.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.8.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.1.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.0.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.6 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.0 - # via - # azure-cosmosdb-table - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in -backports.entry-points-selectable==1.1.0 +backports-entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 # via # paramiko # passlib +boto==2.49.0 + # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto -boto==2.49.0 - # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -367,7 +58,6 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # python-telegram-bot # requests certvalidator==0.11.1 @@ -375,7 +65,6 @@ certvalidator==0.11.1 cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -408,11 +97,7 @@ croniter==0.3.29 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal # ansible-core - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -469,32 +154,22 @@ idna==2.8 immutables==0.15 # via contextvars importlib-metadata==4.6.4 - # via - # -r requirements/static/pkg/linux.in - # backports.entry-points-selectable - # jsonschema - # mako - # moto - # pluggy - # pytest - # virtualenv + # via -r requirements/static/pkg/linux.in iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest -jaraco.classes==3.2.1 - # via jaraco.collections -jaraco.collections==3.4.0 +jaraco-classes==3.2.1 + # via jaraco-collections +jaraco-collections==3.4.0 # via cherrypy -jaraco.functools==2.0 +jaraco-functools==2.0 # via # cheroot - # jaraco.text + # jaraco-text # tempora -jaraco.text==3.5.1 - # via jaraco.collections +jaraco-text==3.5.1 + # via jaraco-collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -516,14 +191,10 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -548,103 +219,14 @@ more-itertools==5.0.0 # via # cheroot # cherrypy - # jaraco.classes - # jaraco.functools + # jaraco-classes + # jaraco-functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.14 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.3 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -662,8 +244,6 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.0 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -671,9 +251,8 @@ packaging==21.3 # -r requirements/base.txt # ansible-core # pytest -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -699,28 +278,27 @@ psutil==5.8.0 # pytest-salt-factories # pytest-shell-utilities # pytest-system-statistics -pyasn1-modules==0.2.4 - # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pycparser==2.17 - # via cffi +pyasn1-modules==0.2.4 + # via google-auth +pycparser==2.21 ; python_version >= "3.9" + # via + # -r requirements/static/ci/common.in + # -r requirements/static/pkg/linux.in + # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt pyeapi==0.8.3 # via napalm -pygit2==1.0.3 ; python_version <= "3.8" +pygit2==1.12.1 ; python_version > "3.8" # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # adal - # twilio + # via twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -739,6 +317,18 @@ pyserial==3.4 # via # junos-eznc # netmiko +pytest==7.2.0 ; python_version > "3.6" + # via + # -r requirements/pytest.txt + # pytest-custom-exit-code + # pytest-helpers-namespace + # pytest-salt-factories + # pytest-shell-utilities + # pytest-skip-markers + # pytest-subtests + # pytest-system-statistics + # pytest-tempdir + # pytest-timeout pytest-custom-exit-code==0.3.0 # via -r requirements/pytest.txt pytest-helpers-namespace==2021.4.29 @@ -767,26 +357,11 @@ pytest-tempdir==2019.10.12 # pytest-salt-factories pytest-timeout==1.4.2 # via -r requirements/pytest.txt -pytest==7.2.0 ; python_version > "3.6" - # via - # -r requirements/pytest.txt - # pytest-custom-exit-code - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-subtests - # pytest-system-statistics - # pytest-tempdir - # pytest-timeout python-consul==1.1.0 # via -r requirements/static/ci/linux.in python-dateutil==2.8.1 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -822,33 +397,22 @@ pyzmq==23.2.0 ; python_version < "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -redis-py-cluster==2.1.3 - # via -r requirements/static/ci/linux.in redis==3.5.3 # via redis-py-cluster -requests-oauthlib==1.3.0 - # via msrest +redis-py-cluster==2.1.3 + # via -r requirements/static/ci/linux.in requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -883,10 +447,8 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes # mock # more-itertools @@ -935,13 +497,8 @@ twilio==7.9.2 # via -r requirements/static/ci/linux.in typing-extensions==3.10.0.0 # via - # aiohttp - # async-timeout - # gitpython - # importlib-metadata # pytest-shell-utilities # pytest-system-statistics - # yarl tzlocal==2.1 # via apscheduler urllib3==1.26.6 @@ -974,7 +531,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==1.4 +zc-lockfile==1.4 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.8/cloud.txt b/requirements/static/ci/py3.8/cloud.txt index 9fa61127c8f..b5a3fb09393 100644 --- a/requirements/static/ci/py3.8/cloud.txt +++ b/requirements/static/ci/py3.8/cloud.txt @@ -4,10 +4,6 @@ # # pip-compile --output-file=requirements/static/ci/py3.8/cloud.txt --pip-args='--constraint=requirements/static/ci/py3.8/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/cloud.in requirements/static/ci/common.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.7 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -31,304 +27,6 @@ attrs==21.2.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.27 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.52 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.2.0 @@ -354,14 +52,12 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # requests certvalidator==0.11.1 # via vcert cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -393,10 +89,6 @@ croniter==1.0.15 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -459,8 +151,6 @@ iniconfig==1.1.1 # via pytest ipaddress==1.0.23 # via kubernetes -isodate==0.6.0 - # via msrest jaraco.classes==3.2.1 # via jaraco.collections jaraco.collections==3.4.0 @@ -492,14 +182,10 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in loguru==0.6.0 # via ciscoconfparse looseversion==1.0.2 @@ -532,98 +218,6 @@ msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.21 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -648,17 +242,14 @@ ntc-templates==2.3.2 # netmiko ntlm-auth==1.3.0 # via requests-ntlm -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.1 # via certvalidator packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -694,10 +285,6 @@ pycryptodomex==3.10.1 # via -r requirements/crypto.txt pyeapi==0.8.4 # via napalm -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in -pyjwt==2.4.0 - # via adal pynacl==1.4.0 # via paramiko pyopenssl==23.0.0 @@ -763,9 +350,6 @@ pytest==7.2.0 ; python_version > "3.6" python-dateutil==2.8.2 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -797,31 +381,20 @@ pyzmq==23.2.0 ; python_version < "3.11" # pytest-salt-factories requests-ntlm==1.1.0 # via pywinrm -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # profitbricks # pyvmomi # pywinrm # requests-ntlm - # requests-oauthlib # responses # vcert responses==0.14.0 @@ -851,12 +424,9 @@ six==1.16.0 # etcd3-py # genshi # geomet - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes - # msrestazure # ncclient # paramiko # profitbricks diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index 213c64652d4..07a1ba0f7b7 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/static/ci/py3.8/freebsd.txt --pip-args='--constraint=requirements/static/pkg/py3.8/freebsd.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/freebsd.in requirements/static/pkg/freebsd.in requirements/zeromq.txt # -adal==1.2.5 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -29,316 +25,18 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.26 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.51 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in -backports.entry-points-selectable==1.1.0 +backports-entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 # via # paramiko # passlib +boto==2.49.0 + # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto -boto==2.49.0 - # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -352,14 +50,12 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # requests certvalidator==0.11.1 # via vcert cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -391,10 +87,6 @@ croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in cryptography==39.0.2 # via - # adal - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -457,19 +149,17 @@ iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest -jaraco.classes==3.2.1 - # via jaraco.collections -jaraco.collections==3.4.0 +jaraco-classes==3.2.1 + # via jaraco-collections +jaraco-collections==3.4.0 # via cherrypy -jaraco.functools==2.0 +jaraco-functools==2.0 # via # cheroot - # jaraco.text + # jaraco-text # tempora -jaraco.text==3.5.1 - # via jaraco.collections +jaraco-text==3.5.1 + # via jaraco-collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -490,14 +180,10 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -522,106 +208,14 @@ more-itertools==5.0.0 # via # cheroot # cherrypy - # jaraco.classes - # jaraco.functools + # jaraco-classes + # jaraco-functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.19 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -639,17 +233,14 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -675,24 +266,23 @@ psutil==5.8.0 # pytest-salt-factories # pytest-shell-utilities # pytest-system-statistics -pyasn1-modules==0.2.4 - # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pycparser==2.17 - # via cffi +pyasn1-modules==0.2.4 + # via google-auth +pycparser==2.21 ; python_version >= "3.9" + # via + # -r requirements/static/ci/common.in + # -r requirements/static/pkg/freebsd.in + # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt pyeapi==0.8.3 # via napalm pygit2==1.8.0 ; python_version >= "3.7" # via -r requirements/static/ci/freebsd.in -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in -pyjwt==2.4.0 - # via adal pynacl==1.3.0 # via paramiko pyopenssl==23.0.0 @@ -709,6 +299,18 @@ pyserial==3.4 # via # junos-eznc # netmiko +pytest==7.2.0 ; python_version > "3.6" + # via + # -r requirements/pytest.txt + # pytest-custom-exit-code + # pytest-helpers-namespace + # pytest-salt-factories + # pytest-shell-utilities + # pytest-skip-markers + # pytest-subtests + # pytest-system-statistics + # pytest-tempdir + # pytest-timeout pytest-custom-exit-code==0.3.0 # via -r requirements/pytest.txt pytest-helpers-namespace==2021.4.29 @@ -737,24 +339,9 @@ pytest-tempdir==2019.10.12 # pytest-salt-factories pytest-timeout==1.4.2 # via -r requirements/pytest.txt -pytest==7.2.0 ; python_version > "3.6" - # via - # -r requirements/pytest.txt - # pytest-custom-exit-code - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-subtests - # pytest-system-statistics - # pytest-tempdir - # pytest-timeout python-dateutil==2.8.1 # via # -r requirements/static/pkg/freebsd.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -783,28 +370,17 @@ pyzmq==23.2.0 ; python_version < "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # pyvmomi - # requests-oauthlib # responses # vcert responses==0.10.6 @@ -833,14 +409,11 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes # mock # more-itertools - # msrestazure # ncclient # paramiko # pynacl @@ -909,7 +482,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==1.4 +zc-lockfile==1.4 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.8/lint.txt b/requirements/static/ci/py3.8/lint.txt index 27832bd6b76..007d6224a0a 100644 --- a/requirements/static/ci/py3.8/lint.txt +++ b/requirements/static/ci/py3.8/lint.txt @@ -1,21 +1,17 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/static/ci/py3.8/lint.txt --pip-args='--constraint=requirements/static/ci/py3.8/linux.txt' requirements/base.txt requirements/static/ci/common.in requirements/static/ci/lint.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.7 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 # via aiohttp -ansible-core==2.11.4 - # via ansible -ansible==4.4.0 ; python_version < "3.9" +ansible==7.5.0 ; python_version >= "3.9" # via -r requirements/static/ci/linux.in +ansible-core==2.14.6 + # via ansible apache-libcloud==3.3.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in apscheduler==3.6.3 @@ -32,318 +28,18 @@ attrs==21.2.0 # via # aiohttp # jsonschema -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.27 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.52 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in -backports.entry-points-selectable==1.1.0 +backports-entry-points-selectable==1.1.0 # via virtualenv -backports.zoneinfo==0.2.1 - # via tzlocal bcrypt==3.2.0 # via # paramiko # passlib +boto==2.49.0 + # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto -boto==2.49.0 - # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -359,7 +55,6 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # python-telegram-bot # requests certvalidator==0.11.1 @@ -367,7 +62,6 @@ certvalidator==0.11.1 cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -400,11 +94,7 @@ croniter==1.0.15 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal # ansible-core - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -456,21 +146,19 @@ importlib-metadata==4.6.4 # via -r requirements/static/pkg/linux.in ipaddress==1.0.23 # via kubernetes -isodate==0.6.0 - # via msrest isort==4.3.21 # via pylint -jaraco.classes==3.2.1 - # via jaraco.collections -jaraco.collections==3.4.0 +jaraco-classes==3.2.1 + # via jaraco-collections +jaraco-collections==3.4.0 # via cherrypy -jaraco.functools==3.3.0 +jaraco-functools==3.3.0 # via # cheroot - # jaraco.text + # jaraco-text # tempora -jaraco.text==3.5.1 - # via jaraco.collections +jaraco-text==3.5.1 + # via jaraco-collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -492,16 +180,12 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in lazy-object-proxy==1.4.3 # via astroid -libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in loguru==0.6.0 # via ciscoconfparse looseversion==1.0.2 @@ -530,104 +214,12 @@ more-itertools==8.8.0 # via # cheroot # cherrypy - # jaraco.classes - # jaraco.functools + # jaraco-classes + # jaraco-functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 # via -r requirements/base.txt -msrest==0.6.21 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -649,17 +241,14 @@ ntc-templates==2.2.2 # via # junos-eznc # netmiko -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.1 # via certvalidator packaging==21.3 # via # -r requirements/base.txt # ansible-core -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -677,30 +266,29 @@ portend==2.7.1 # via cherrypy psutil==5.8.0 # via -r requirements/base.txt -pyasn1-modules==0.2.8 - # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa +pyasn1-modules==0.2.8 + # via google-auth pycodestyle==2.5.0 # via saltpylint -pycparser==2.20 - # via cffi +pycparser==2.21 ; python_version >= "3.9" + # via + # -r requirements/static/ci/common.in + # -r requirements/static/pkg/linux.in + # cffi pycryptodomex==3.10.1 # via -r requirements/crypto.txt pyeapi==0.8.4 # via napalm -pygit2==1.0.3 ; python_version <= "3.8" +pygit2==1.12.1 ; python_version > "3.8" # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # adal - # twilio + # via twilio pylint==2.4.4 # via # -r requirements/static/ci/lint.in @@ -728,9 +316,6 @@ python-consul==1.1.0 python-dateutil==2.8.2 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -763,33 +348,22 @@ pyyaml==5.4.1 # yamlordereddictloader pyzmq==23.2.0 ; python_version < "3.11" # via -r requirements/zeromq.txt -redis-py-cluster==2.1.3 - # via -r requirements/static/ci/linux.in redis==3.5.3 # via redis-py-cluster -requests-oauthlib==1.3.0 - # via msrest +redis-py-cluster==2.1.3 + # via -r requirements/static/ci/linux.in requests==2.26.0 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -826,12 +400,9 @@ six==1.16.0 # etcd3-py # genshi # geomet - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes - # msrestazure # ncclient # paramiko # pynacl @@ -905,7 +476,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==2.0 +zc-lockfile==2.0 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.8/linux.txt b/requirements/static/ci/py3.8/linux.txt index e7fd9969871..e4bc957798d 100644 --- a/requirements/static/ci/py3.8/linux.txt +++ b/requirements/static/ci/py3.8/linux.txt @@ -1,21 +1,17 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/static/ci/py3.8/linux.txt --pip-args='--constraint=requirements/static/pkg/py3.8/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.3 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 # via aiohttp -ansible-core==2.11.7 - # via ansible -ansible==4.4.0 ; python_version < "3.9" +ansible==7.5.0 ; python_version >= "3.9" # via -r requirements/static/ci/linux.in +ansible-core==2.14.6 + # via ansible apache-libcloud==2.5.0 ; sys_platform != "win32" # via -r requirements/static/ci/common.in apscheduler==3.6.3 @@ -35,321 +31,18 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.18 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.5 - # via azure -azure-datalake-store==0.0.44 - # via azure -azure-eventgrid==1.2.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.0 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.4.1 - # via azure-mgmt -azure-mgmt-containerregistry==2.7.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.5.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.0 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.6.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.0.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.8.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.1.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.0.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.6 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.0 - # via - # azure-cosmosdb-table - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in -backports.entry-points-selectable==1.1.0 +backports-entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 # via # paramiko # passlib +boto==2.49.0 + # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto -boto==2.49.0 - # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -365,7 +58,6 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # python-telegram-bot # requests certvalidator==0.11.1 @@ -373,7 +65,6 @@ certvalidator==0.11.1 cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -406,11 +97,7 @@ croniter==0.3.29 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal # ansible-core - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -472,19 +159,17 @@ iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest -jaraco.classes==3.2.1 - # via jaraco.collections -jaraco.collections==3.4.0 +jaraco-classes==3.2.1 + # via jaraco-collections +jaraco-collections==3.4.0 # via cherrypy -jaraco.functools==2.0 +jaraco-functools==2.0 # via # cheroot - # jaraco.text + # jaraco-text # tempora -jaraco.text==3.5.1 - # via jaraco.collections +jaraco-text==3.5.1 + # via jaraco-collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -506,14 +191,10 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -538,103 +219,14 @@ more-itertools==5.0.0 # via # cheroot # cherrypy - # jaraco.classes - # jaraco.functools + # jaraco-classes + # jaraco-functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.14 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.3 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -652,8 +244,6 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==22.0 @@ -661,9 +251,8 @@ packaging==22.0 # -r requirements/base.txt # ansible-core # pytest -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -689,28 +278,27 @@ psutil==5.8.0 # pytest-salt-factories # pytest-shell-utilities # pytest-system-statistics -pyasn1-modules==0.2.4 - # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pycparser==2.17 - # via cffi +pyasn1-modules==0.2.4 + # via google-auth +pycparser==2.21 ; python_version >= "3.9" + # via + # -r requirements/static/ci/common.in + # -r requirements/static/pkg/linux.in + # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt pyeapi==0.8.3 # via napalm -pygit2==1.0.3 ; python_version <= "3.8" +pygit2==1.12.1 ; python_version > "3.8" # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # adal - # twilio + # via twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -727,6 +315,18 @@ pyserial==3.4 # via # junos-eznc # netmiko +pytest==7.2.0 ; python_version > "3.6" + # via + # -r requirements/pytest.txt + # pytest-custom-exit-code + # pytest-helpers-namespace + # pytest-salt-factories + # pytest-shell-utilities + # pytest-skip-markers + # pytest-subtests + # pytest-system-statistics + # pytest-tempdir + # pytest-timeout pytest-custom-exit-code==0.3.0 # via -r requirements/pytest.txt pytest-helpers-namespace==2021.4.29 @@ -755,26 +355,11 @@ pytest-tempdir==2019.10.12 # pytest-salt-factories pytest-timeout==1.4.2 # via -r requirements/pytest.txt -pytest==7.2.0 ; python_version > "3.6" - # via - # -r requirements/pytest.txt - # pytest-custom-exit-code - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-subtests - # pytest-system-statistics - # pytest-tempdir - # pytest-timeout python-consul==1.1.0 # via -r requirements/static/ci/linux.in python-dateutil==2.8.1 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -810,33 +395,22 @@ pyzmq==23.2.0 ; python_version < "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -redis-py-cluster==2.1.3 - # via -r requirements/static/ci/linux.in redis==3.5.3 # via redis-py-cluster -requests-oauthlib==1.3.0 - # via msrest +redis-py-cluster==2.1.3 + # via -r requirements/static/ci/linux.in requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -871,10 +445,8 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes # mock # more-itertools @@ -957,7 +529,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==1.4 +zc-lockfile==1.4 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.9/cloud.txt b/requirements/static/ci/py3.9/cloud.txt index fc91bb9d9ad..f283439bf82 100644 --- a/requirements/static/ci/py3.9/cloud.txt +++ b/requirements/static/ci/py3.9/cloud.txt @@ -4,10 +4,6 @@ # # pip-compile --output-file=requirements/static/ci/py3.9/cloud.txt --pip-args='--constraint=requirements/static/ci/py3.9/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/cloud.in requirements/static/ci/common.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.7 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -31,304 +27,6 @@ attrs==21.2.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.27 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.52 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.2.0 @@ -354,14 +52,12 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # requests certvalidator==0.11.1 # via vcert cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -393,10 +89,6 @@ croniter==1.0.15 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -459,8 +151,6 @@ iniconfig==1.1.1 # via pytest ipaddress==1.0.23 # via kubernetes -isodate==0.6.0 - # via msrest jaraco.classes==3.2.1 # via jaraco.collections jaraco.collections==3.4.0 @@ -492,14 +182,10 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in loguru==0.6.0 # via ciscoconfparse looseversion==1.0.2 @@ -532,98 +218,6 @@ msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.21 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -648,17 +242,14 @@ ntc-templates==2.3.2 # netmiko ntlm-auth==1.3.0 # via requests-ntlm -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.1 # via certvalidator packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -697,10 +288,6 @@ pycryptodomex==3.10.1 # via -r requirements/crypto.txt pyeapi==0.8.4 # via napalm -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in -pyjwt==2.4.0 - # via adal pynacl==1.4.0 # via paramiko pyopenssl==23.0.0 @@ -766,9 +353,6 @@ pytest==7.2.0 ; python_version > "3.6" python-dateutil==2.8.2 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -800,31 +384,20 @@ pyzmq==23.2.0 ; python_version < "3.11" # pytest-salt-factories requests-ntlm==1.1.0 # via pywinrm -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # profitbricks # pyvmomi # pywinrm # requests-ntlm - # requests-oauthlib # responses # vcert responses==0.14.0 @@ -854,12 +427,9 @@ six==1.16.0 # etcd3-py # genshi # geomet - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes - # msrestazure # ncclient # paramiko # profitbricks diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index 8be41123871..080fa1d9f99 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -4,10 +4,6 @@ # # pip-compile --output-file=requirements/static/ci/py3.9/darwin.txt --pip-args='--constraint=requirements/static/pkg/py3.9/darwin.txt' requirements/darwin.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/darwin.in requirements/static/pkg/darwin.in # -adal==1.2.5 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -31,304 +27,6 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.26 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.51 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 @@ -354,14 +52,12 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # requests certvalidator==0.11.1 # via vcert cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -394,10 +90,6 @@ croniter==0.3.29 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/darwin.txt - # adal - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -462,8 +154,6 @@ iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest jaraco.classes==3.2.1 # via jaraco.collections jaraco.collections==3.4.0 @@ -533,98 +223,6 @@ msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.19 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -642,8 +240,6 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -695,8 +291,6 @@ pyeapi==0.8.3 # via napalm pygit2==1.9.1 ; python_version >= "3.7" # via -r requirements/static/ci/darwin.in -pyjwt==2.4.0 - # via adal pynacl==1.3.0 # via paramiko pyopenssl==23.0.0 @@ -756,9 +350,6 @@ pytest==7.2.0 ; python_version > "3.6" python-dateutil==2.8.0 # via # -r requirements/darwin.txt - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -787,28 +378,17 @@ pyzmq==23.2.0 ; python_version < "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # pyvmomi - # requests-oauthlib # responses # vcert # vultr @@ -838,12 +418,10 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kubernetes # mock - # msrestazure # ncclient # paramiko # pynacl diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index 59fa8c6b367..f24fc5d81bb 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/static/ci/py3.9/freebsd.txt --pip-args='--constraint=requirements/static/pkg/py3.9/freebsd.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/freebsd.in requirements/static/pkg/freebsd.in requirements/zeromq.txt # -adal==1.2.5 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 @@ -29,316 +25,18 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.26 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.51 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in -backports.entry-points-selectable==1.1.0 +backports-entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 # via # paramiko # passlib +boto==2.49.0 + # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto -boto==2.49.0 - # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -352,14 +50,12 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # requests certvalidator==0.11.1 # via vcert cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -391,10 +87,6 @@ croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in cryptography==39.0.2 # via - # adal - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -457,19 +149,17 @@ iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest -jaraco.classes==3.2.1 - # via jaraco.collections -jaraco.collections==3.4.0 +jaraco-classes==3.2.1 + # via jaraco-collections +jaraco-collections==3.4.0 # via cherrypy -jaraco.functools==2.0 +jaraco-functools==2.0 # via # cheroot - # jaraco.text + # jaraco-text # tempora -jaraco.text==3.5.1 - # via jaraco.collections +jaraco-text==3.5.1 + # via jaraco-collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -490,14 +180,10 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -522,106 +208,14 @@ more-itertools==5.0.0 # via # cheroot # cherrypy - # jaraco.classes - # jaraco.functools + # jaraco-classes + # jaraco-functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.19 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -639,17 +233,14 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -675,12 +266,12 @@ psutil==5.8.0 # pytest-salt-factories # pytest-shell-utilities # pytest-system-statistics -pyasn1-modules==0.2.4 - # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa +pyasn1-modules==0.2.4 + # via google-auth pycparser==2.21 ; python_version >= "3.9" # via # -r requirements/static/ci/common.in @@ -692,10 +283,6 @@ pyeapi==0.8.3 # via napalm pygit2==1.8.0 ; python_version >= "3.7" # via -r requirements/static/ci/freebsd.in -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in -pyjwt==2.4.0 - # via adal pynacl==1.3.0 # via paramiko pyopenssl==23.0.0 @@ -712,6 +299,18 @@ pyserial==3.4 # via # junos-eznc # netmiko +pytest==7.2.0 ; python_version > "3.6" + # via + # -r requirements/pytest.txt + # pytest-custom-exit-code + # pytest-helpers-namespace + # pytest-salt-factories + # pytest-shell-utilities + # pytest-skip-markers + # pytest-subtests + # pytest-system-statistics + # pytest-tempdir + # pytest-timeout pytest-custom-exit-code==0.3.0 # via -r requirements/pytest.txt pytest-helpers-namespace==2021.4.29 @@ -740,24 +339,9 @@ pytest-tempdir==2019.10.12 # pytest-salt-factories pytest-timeout==1.4.2 # via -r requirements/pytest.txt -pytest==7.2.0 ; python_version > "3.6" - # via - # -r requirements/pytest.txt - # pytest-custom-exit-code - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-subtests - # pytest-system-statistics - # pytest-tempdir - # pytest-timeout python-dateutil==2.8.1 # via # -r requirements/static/pkg/freebsd.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -786,28 +370,17 @@ pyzmq==23.2.0 ; python_version < "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # pyvmomi - # requests-oauthlib # responses # vcert responses==0.10.6 @@ -836,14 +409,11 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes # mock # more-itertools - # msrestazure # ncclient # paramiko # pynacl @@ -912,7 +482,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==1.4 +zc-lockfile==1.4 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.9/lint.txt b/requirements/static/ci/py3.9/lint.txt index 51cf5153aed..e2a83aa2d99 100644 --- a/requirements/static/ci/py3.9/lint.txt +++ b/requirements/static/ci/py3.9/lint.txt @@ -1,21 +1,17 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/static/ci/py3.9/lint.txt --pip-args='--constraint=requirements/static/ci/py3.9/linux.txt' requirements/base.txt requirements/static/ci/common.in requirements/static/ci/lint.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.7 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 # via aiohttp -ansible-core==2.14.1 - # via ansible ansible==7.1.0 ; python_version >= "3.9" # via -r requirements/static/ci/linux.in +ansible-core==2.14.1 + # via ansible apache-libcloud==3.3.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in apscheduler==3.6.3 @@ -32,316 +28,18 @@ attrs==21.2.0 # via # aiohttp # jsonschema -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.27 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.6 - # via azure -azure-datalake-store==0.0.52 - # via azure -azure-eventgrid==1.3.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.2 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.5.0 - # via azure-mgmt -azure-mgmt-containerregistry==2.8.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.6.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.1 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.7.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.1.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.9.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.2.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.1.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.7 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.2 - # via - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in -backports.entry-points-selectable==1.1.0 +backports-entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.2.0 # via # paramiko # passlib +boto==2.49.0 + # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto -boto==2.49.0 - # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -357,7 +55,6 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # python-telegram-bot # requests certvalidator==0.11.1 @@ -365,7 +62,6 @@ certvalidator==0.11.1 cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -398,11 +94,7 @@ croniter==1.0.15 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal # ansible-core - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -454,21 +146,19 @@ importlib-metadata==6.0.0 # via -r requirements/static/pkg/linux.in ipaddress==1.0.23 # via kubernetes -isodate==0.6.0 - # via msrest isort==4.3.21 # via pylint -jaraco.classes==3.2.1 - # via jaraco.collections -jaraco.collections==3.4.0 +jaraco-classes==3.2.1 + # via jaraco-collections +jaraco-collections==3.4.0 # via cherrypy -jaraco.functools==3.3.0 +jaraco-functools==3.3.0 # via # cheroot - # jaraco.text + # jaraco-text # tempora -jaraco.text==3.5.1 - # via jaraco.collections +jaraco-text==3.5.1 + # via jaraco-collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -490,16 +180,12 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in lazy-object-proxy==1.4.3 # via astroid -libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in loguru==0.6.0 # via ciscoconfparse looseversion==1.0.2 @@ -528,104 +214,12 @@ more-itertools==8.8.0 # via # cheroot # cherrypy - # jaraco.classes - # jaraco.functools + # jaraco-classes + # jaraco-functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 # via -r requirements/base.txt -msrest==0.6.21 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-managementpartner - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.4 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -647,17 +241,14 @@ ntc-templates==2.2.2 # via # junos-eznc # netmiko -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.1 # via certvalidator packaging==21.3 # via # -r requirements/base.txt # ansible-core -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -675,12 +266,12 @@ portend==2.7.1 # via cherrypy psutil==5.8.0 # via -r requirements/base.txt -pyasn1-modules==0.2.8 - # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa +pyasn1-modules==0.2.8 + # via google-auth pycodestyle==2.5.0 # via saltpylint pycparser==2.21 ; python_version >= "3.9" @@ -696,12 +287,8 @@ pygit2==1.6.1 ; python_version > "3.8" # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # adal - # twilio + # via twilio pylint==2.4.4 # via # -r requirements/static/ci/lint.in @@ -729,9 +316,6 @@ python-consul==1.1.0 python-dateutil==2.8.2 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -764,33 +348,22 @@ pyyaml==5.4.1 # yamlordereddictloader pyzmq==23.2.0 ; python_version < "3.11" # via -r requirements/zeromq.txt -redis-py-cluster==2.1.3 - # via -r requirements/static/ci/linux.in redis==3.5.3 # via redis-py-cluster -requests-oauthlib==1.3.0 - # via msrest +redis-py-cluster==2.1.3 + # via -r requirements/static/ci/linux.in requests==2.26.0 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -827,12 +400,9 @@ six==1.16.0 # etcd3-py # genshi # geomet - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes - # msrestazure # ncclient # paramiko # pynacl @@ -906,7 +476,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==2.0 +zc-lockfile==2.0 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index 0821935f033..e5fbc53e96e 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -1,21 +1,17 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/static/ci/py3.9/linux.txt --pip-args='--constraint=requirements/static/pkg/py3.9/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # -adal==1.2.3 - # via - # azure-datalake-store - # msrestazure aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 # via aiohttp -ansible-core==2.14.1 - # via ansible ansible==7.1.0 ; python_version >= "3.9" # via -r requirements/static/ci/linux.in +ansible-core==2.14.1 + # via ansible apache-libcloud==2.5.0 ; sys_platform != "win32" # via -r requirements/static/ci/common.in apscheduler==3.6.3 @@ -35,321 +31,18 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -azure-applicationinsights==0.1.0 - # via azure -azure-batch==4.1.3 - # via azure -azure-common==1.1.18 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-table - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy - # azure-storage-blob - # azure-storage-common - # azure-storage-file - # azure-storage-queue -azure-cosmosdb-nspkg==2.0.2 - # via azure-cosmosdb-table -azure-cosmosdb-table==1.0.5 - # via azure -azure-datalake-store==0.0.44 - # via azure -azure-eventgrid==1.2.0 - # via azure -azure-graphrbac==0.40.0 - # via azure -azure-keyvault==1.1.0 - # via azure -azure-loganalytics==0.1.0 - # via azure -azure-mgmt-advisor==1.0.1 - # via azure-mgmt -azure-mgmt-applicationinsights==0.1.1 - # via azure-mgmt -azure-mgmt-authorization==0.50.0 - # via azure-mgmt -azure-mgmt-batch==5.0.1 - # via azure-mgmt -azure-mgmt-batchai==2.0.0 - # via azure-mgmt -azure-mgmt-billing==0.2.0 - # via azure-mgmt -azure-mgmt-cdn==3.1.0 - # via azure-mgmt -azure-mgmt-cognitiveservices==3.0.0 - # via azure-mgmt -azure-mgmt-commerce==1.0.1 - # via azure-mgmt -azure-mgmt-compute==4.6.0 - # via azure-mgmt -azure-mgmt-consumption==2.0.0 - # via azure-mgmt -azure-mgmt-containerinstance==1.4.1 - # via azure-mgmt -azure-mgmt-containerregistry==2.7.0 - # via azure-mgmt -azure-mgmt-containerservice==4.4.0 - # via azure-mgmt -azure-mgmt-cosmosdb==0.4.1 - # via azure-mgmt -azure-mgmt-datafactory==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-analytics==0.6.0 - # via azure-mgmt -azure-mgmt-datalake-nspkg==3.0.1 - # via - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store -azure-mgmt-datalake-store==0.5.0 - # via azure-mgmt -azure-mgmt-datamigration==1.0.0 - # via azure-mgmt -azure-mgmt-devspaces==0.1.0 - # via azure-mgmt -azure-mgmt-devtestlabs==2.2.0 - # via azure-mgmt -azure-mgmt-dns==2.1.0 - # via azure-mgmt -azure-mgmt-eventgrid==1.0.0 - # via azure-mgmt -azure-mgmt-eventhub==2.5.0 - # via azure-mgmt -azure-mgmt-hanaonazure==0.1.1 - # via azure-mgmt -azure-mgmt-iotcentral==0.1.0 - # via azure-mgmt -azure-mgmt-iothub==0.5.0 - # via azure-mgmt -azure-mgmt-iothubprovisioningservices==0.2.0 - # via azure-mgmt -azure-mgmt-keyvault==1.1.0 - # via azure-mgmt -azure-mgmt-loganalytics==0.2.0 - # via azure-mgmt -azure-mgmt-logic==3.0.0 - # via azure-mgmt -azure-mgmt-machinelearningcompute==0.4.1 - # via azure-mgmt -azure-mgmt-managementgroups==0.1.0 - # via azure-mgmt -azure-mgmt-managementpartner==0.1.0 - # via azure-mgmt -azure-mgmt-maps==0.1.0 - # via azure-mgmt -azure-mgmt-marketplaceordering==0.1.0 - # via azure-mgmt -azure-mgmt-media==1.0.0 - # via azure-mgmt -azure-mgmt-monitor==0.5.2 - # via azure-mgmt -azure-mgmt-msi==0.2.0 - # via azure-mgmt -azure-mgmt-network==2.6.0 - # via azure-mgmt -azure-mgmt-notificationhubs==2.0.0 - # via azure-mgmt -azure-mgmt-nspkg==3.0.2 - # via - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-consumption - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-nspkg - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web -azure-mgmt-policyinsights==0.1.0 - # via azure-mgmt -azure-mgmt-powerbiembedded==2.0.0 - # via azure-mgmt -azure-mgmt-rdbms==1.8.0 - # via azure-mgmt -azure-mgmt-recoveryservices==0.3.0 - # via azure-mgmt -azure-mgmt-recoveryservicesbackup==0.3.0 - # via azure-mgmt -azure-mgmt-redis==5.0.0 - # via azure-mgmt -azure-mgmt-relay==0.1.0 - # via azure-mgmt -azure-mgmt-reservations==0.2.1 - # via azure-mgmt -azure-mgmt-resource==2.1.0 - # via azure-mgmt -azure-mgmt-scheduler==2.0.0 - # via azure-mgmt -azure-mgmt-search==2.0.0 - # via azure-mgmt -azure-mgmt-servicebus==0.5.3 - # via azure-mgmt -azure-mgmt-servicefabric==0.2.0 - # via azure-mgmt -azure-mgmt-signalr==0.1.1 - # via azure-mgmt -azure-mgmt-sql==0.9.1 - # via azure-mgmt -azure-mgmt-storage==2.0.0 - # via azure-mgmt -azure-mgmt-subscription==0.2.0 - # via azure-mgmt -azure-mgmt-trafficmanager==0.50.0 - # via azure-mgmt -azure-mgmt-web==0.35.0 - # via azure-mgmt -azure-mgmt==4.0.0 - # via azure -azure-nspkg==3.0.2 - # via - # azure-applicationinsights - # azure-batch - # azure-cosmosdb-nspkg - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-loganalytics - # azure-mgmt-nspkg - # azure-servicebus - # azure-servicefabric - # azure-servicemanagement-legacy -azure-servicebus==0.21.1 - # via azure -azure-servicefabric==6.3.0.0 - # via azure -azure-servicemanagement-legacy==0.20.6 - # via azure -azure-storage-blob==1.5.0 - # via azure -azure-storage-common==1.4.0 - # via - # azure-cosmosdb-table - # azure-storage-blob - # azure-storage-file - # azure-storage-queue -azure-storage-file==1.4.0 - # via azure -azure-storage-queue==1.4.0 - # via azure -azure==4.0.0 ; sys_platform != "win32" - # via -r requirements/static/ci/common.in -backports.entry-points-selectable==1.1.0 +backports-entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 # via # paramiko # passlib +boto==2.49.0 + # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto -boto==2.49.0 - # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -367,7 +60,6 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # python-telegram-bot # requests certvalidator==0.11.1 @@ -375,7 +67,6 @@ certvalidator==0.11.1 cffi==1.14.6 # via # -r requirements/static/ci/common.in - # azure-datalake-store # bcrypt # cryptography # napalm @@ -408,11 +99,7 @@ croniter==0.3.29 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal # ansible-core - # azure-cosmosdb-table - # azure-keyvault - # azure-storage-common # etcd3-py # moto # paramiko @@ -474,19 +161,17 @@ iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest -jaraco.classes==3.2.1 - # via jaraco.collections -jaraco.collections==3.4.0 +jaraco-classes==3.2.1 + # via jaraco-collections +jaraco-collections==3.4.0 # via cherrypy -jaraco.functools==2.0 +jaraco-functools==2.0 # via # cheroot - # jaraco.text + # jaraco-text # tempora -jaraco.text==3.5.1 - # via jaraco.collections +jaraco-text==3.5.1 + # via jaraco-collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -508,14 +193,10 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin" - # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -540,103 +221,14 @@ more-itertools==5.0.0 # via # cheroot # cherrypy - # jaraco.classes - # jaraco.functools + # jaraco-classes + # jaraco-functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 # via # -r requirements/base.txt # pytest-salt-factories -msrest==0.6.14 - # via - # azure-applicationinsights - # azure-eventgrid - # azure-keyvault - # azure-loganalytics - # azure-mgmt-cdn - # azure-mgmt-compute - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-dns - # azure-mgmt-eventhub - # azure-mgmt-keyvault - # azure-mgmt-media - # azure-mgmt-network - # azure-mgmt-rdbms - # azure-mgmt-resource - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-servicefabric - # msrestazure -msrestazure==0.6.3 - # via - # azure-batch - # azure-eventgrid - # azure-graphrbac - # azure-keyvault - # azure-mgmt-advisor - # azure-mgmt-applicationinsights - # azure-mgmt-authorization - # azure-mgmt-batch - # azure-mgmt-batchai - # azure-mgmt-billing - # azure-mgmt-cdn - # azure-mgmt-cognitiveservices - # azure-mgmt-commerce - # azure-mgmt-compute - # azure-mgmt-consumption - # azure-mgmt-containerinstance - # azure-mgmt-containerregistry - # azure-mgmt-containerservice - # azure-mgmt-cosmosdb - # azure-mgmt-datafactory - # azure-mgmt-datalake-analytics - # azure-mgmt-datalake-store - # azure-mgmt-datamigration - # azure-mgmt-devspaces - # azure-mgmt-devtestlabs - # azure-mgmt-dns - # azure-mgmt-eventgrid - # azure-mgmt-eventhub - # azure-mgmt-hanaonazure - # azure-mgmt-iotcentral - # azure-mgmt-iothub - # azure-mgmt-iothubprovisioningservices - # azure-mgmt-keyvault - # azure-mgmt-loganalytics - # azure-mgmt-logic - # azure-mgmt-machinelearningcompute - # azure-mgmt-managementgroups - # azure-mgmt-managementpartner - # azure-mgmt-maps - # azure-mgmt-marketplaceordering - # azure-mgmt-media - # azure-mgmt-monitor - # azure-mgmt-msi - # azure-mgmt-network - # azure-mgmt-notificationhubs - # azure-mgmt-policyinsights - # azure-mgmt-powerbiembedded - # azure-mgmt-rdbms - # azure-mgmt-recoveryservices - # azure-mgmt-recoveryservicesbackup - # azure-mgmt-redis - # azure-mgmt-relay - # azure-mgmt-reservations - # azure-mgmt-resource - # azure-mgmt-scheduler - # azure-mgmt-search - # azure-mgmt-servicebus - # azure-mgmt-servicefabric - # azure-mgmt-signalr - # azure-mgmt-sql - # azure-mgmt-storage - # azure-mgmt-subscription - # azure-mgmt-trafficmanager - # azure-mgmt-web multidict==6.0.2 # via # aiohttp @@ -654,8 +246,6 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==22.0 @@ -663,9 +253,8 @@ packaging==22.0 # -r requirements/base.txt # ansible-core # pytest -paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" +paramiko==2.10.1 # via - # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -691,12 +280,12 @@ psutil==5.8.0 # pytest-salt-factories # pytest-shell-utilities # pytest-system-statistics -pyasn1-modules==0.2.4 - # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa +pyasn1-modules==0.2.4 + # via google-auth pycparser==2.21 ; python_version >= "3.9" # via # -r requirements/static/ci/common.in @@ -710,12 +299,8 @@ pygit2==1.5.0 ; python_version > "3.8" # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in -pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" - # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # adal - # twilio + # via twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -732,6 +317,18 @@ pyserial==3.4 # via # junos-eznc # netmiko +pytest==7.2.0 ; python_version > "3.6" + # via + # -r requirements/pytest.txt + # pytest-custom-exit-code + # pytest-helpers-namespace + # pytest-salt-factories + # pytest-shell-utilities + # pytest-skip-markers + # pytest-subtests + # pytest-system-statistics + # pytest-tempdir + # pytest-timeout pytest-custom-exit-code==0.3.0 # via -r requirements/pytest.txt pytest-helpers-namespace==2021.4.29 @@ -760,26 +357,11 @@ pytest-tempdir==2019.10.12 # pytest-salt-factories pytest-timeout==1.4.2 # via -r requirements/pytest.txt -pytest==7.2.0 ; python_version > "3.6" - # via - # -r requirements/pytest.txt - # pytest-custom-exit-code - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-subtests - # pytest-system-statistics - # pytest-tempdir - # pytest-timeout python-consul==1.1.0 # via -r requirements/static/ci/linux.in python-dateutil==2.8.1 # via # -r requirements/static/pkg/linux.in - # adal - # azure-cosmosdb-table - # azure-storage-common # botocore # croniter # kubernetes @@ -815,33 +397,22 @@ pyzmq==23.2.0 ; python_version < "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -redis-py-cluster==2.1.3 - # via -r requirements/static/ci/linux.in redis==3.5.3 # via redis-py-cluster -requests-oauthlib==1.3.0 - # via msrest +redis-py-cluster==2.1.3 + # via -r requirements/static/ci/linux.in requests==2.25.1 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud - # azure-cosmosdb-table - # azure-datalake-store - # azure-keyvault - # azure-servicebus - # azure-servicemanagement-legacy - # azure-storage-common # docker # etcd3-py # kubernetes # moto - # msrest # napalm # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -876,10 +447,8 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc - # kazoo # kubernetes # mock # more-itertools @@ -962,7 +531,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==1.4 +zc-lockfile==1.4 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 781c5637409..90cf5947776 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -538,7 +538,6 @@ VALID_OPTS = immutabletypes.freeze( "proxy_keep_alive_interval": int, # Update intervals "roots_update_interval": int, - "azurefs_update_interval": int, "gitfs_update_interval": int, "git_pillar_update_interval": int, "hgfs_update_interval": int, @@ -1093,7 +1092,6 @@ DEFAULT_MINION_OPTS = immutabletypes.freeze( "gpg_decrypt_must_succeed": False, # Update intervals "roots_update_interval": DEFAULT_INTERVAL, - "azurefs_update_interval": DEFAULT_INTERVAL, "gitfs_update_interval": DEFAULT_INTERVAL, "git_pillar_update_interval": DEFAULT_INTERVAL, "hgfs_update_interval": DEFAULT_INTERVAL, @@ -1346,7 +1344,6 @@ DEFAULT_MASTER_OPTS = immutabletypes.freeze( "local": True, # Update intervals "roots_update_interval": DEFAULT_INTERVAL, - "azurefs_update_interval": DEFAULT_INTERVAL, "gitfs_update_interval": DEFAULT_INTERVAL, "git_pillar_update_interval": DEFAULT_INTERVAL, "hgfs_update_interval": DEFAULT_INTERVAL, From e2d55c53229ad4d0538a1f62bbc8425dcf2c692e Mon Sep 17 00:00:00 2001 From: nicholasmhughes Date: Mon, 22 May 2023 16:45:26 -0400 Subject: [PATCH 044/152] update reqs --- requirements/static/ci/py3.10/cloud.txt | 10 +- requirements/static/ci/py3.10/freebsd.txt | 12 --- requirements/static/ci/py3.10/lint.txt | 15 +-- requirements/static/ci/py3.10/linux.txt | 14 +-- requirements/static/ci/py3.11/freebsd.txt | 12 --- requirements/static/ci/py3.11/linux.txt | 15 +-- requirements/static/ci/py3.7/cloud.txt | 10 +- requirements/static/ci/py3.7/docs.txt | 17 +--- requirements/static/ci/py3.7/freebsd.txt | 107 ++++++++++++++-------- requirements/static/ci/py3.8/cloud.txt | 10 +- requirements/static/ci/py3.8/docs.txt | 17 +--- requirements/static/ci/py3.8/freebsd.txt | 88 ++++++++++-------- requirements/static/ci/py3.9/cloud.txt | 10 +- requirements/static/ci/py3.9/docs.txt | 27 ++++-- requirements/static/ci/py3.9/freebsd.txt | 75 +++++++++------ requirements/static/ci/py3.9/lint.txt | 60 +++++++----- requirements/static/ci/py3.9/linux.txt | 83 ++++++++++------- 17 files changed, 320 insertions(+), 262 deletions(-) diff --git a/requirements/static/ci/py3.10/cloud.txt b/requirements/static/ci/py3.10/cloud.txt index 669ec1f31f4..1a4b4e1d914 100644 --- a/requirements/static/ci/py3.10/cloud.txt +++ b/requirements/static/ci/py3.10/cloud.txt @@ -169,10 +169,14 @@ junos-eznc==2.6.0 ; sys_platform != "win32" and python_version <= "3.10" # via -r requirements/static/ci/common.in jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in +libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -220,8 +224,9 @@ packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # ncclient # scp @@ -254,6 +259,8 @@ pycparser==2.21 ; python_version >= "3.9" # cffi pycryptodomex==3.10.1 # via -r requirements/crypto.txt +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pynacl==1.4.0 # via paramiko pyopenssl==23.0.0 @@ -388,6 +395,7 @@ six==1.16.0 # geomet # jsonschema # junos-eznc + # kazoo # kubernetes # ncclient # paramiko diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index bf080132823..4dce26c95a1 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -50,7 +50,6 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # requests certvalidator==0.11.1 # via vcert @@ -140,8 +139,6 @@ iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest jaraco.classes==3.2.1 # via jaraco.collections jaraco.collections==3.4.0 @@ -217,8 +214,6 @@ ncclient==0.6.9 # via junos-eznc netaddr==0.7.19 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -266,8 +261,6 @@ pygit2==1.8.0 ; python_version >= "3.7" # via -r requirements/static/ci/freebsd.in pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in -pyjwt==2.4.0 - # via adal pynacl==1.3.0 # via paramiko pyopenssl==23.0.0 @@ -352,8 +345,6 @@ pyzmq==23.2.0 ; python_version < "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt @@ -363,9 +354,7 @@ requests==2.25.1 # etcd3-py # kubernetes # moto - # msrest # pyvmomi - # requests-oauthlib # responses # vcert responses==0.10.6 @@ -391,7 +380,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kazoo diff --git a/requirements/static/ci/py3.10/lint.txt b/requirements/static/ci/py3.10/lint.txt index d5936509f46..a53f8b18c27 100644 --- a/requirements/static/ci/py3.10/lint.txt +++ b/requirements/static/ci/py3.10/lint.txt @@ -55,7 +55,6 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # python-telegram-bot # requests certvalidator==0.11.1 @@ -137,8 +136,6 @@ importlib-metadata==6.0.0 # via -r requirements/static/pkg/linux.in ipaddress==1.0.23 # via kubernetes -isodate==0.6.0 - # via msrest isort==4.3.21 # via pylint jaraco.classes==3.2.1 @@ -219,8 +216,6 @@ ncclient==0.6.9 # via junos-eznc netaddr==0.8.0 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.1 # via certvalidator packaging==21.3 @@ -265,9 +260,7 @@ pyiface==0.0.11 pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # adal - # twilio + # via twilio pylint==2.4.4 # via # -r requirements/static/ci/lint.in @@ -328,22 +321,17 @@ redis-py-cluster==2.1.3 # via -r requirements/static/ci/linux.in redis==3.5.3 # via redis-py-cluster -requests-oauthlib==1.3.0 - # via msrest requests==2.26.0 # via # -r requirements/base.txt # -r requirements/static/ci/common.in - # adal # apache-libcloud # docker # etcd3-py # kubernetes # moto - # msrest # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -377,7 +365,6 @@ six==1.16.0 # etcd3-py # genshi # geomet - # isodate # jsonschema # junos-eznc # kazoo diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index 040450adf01..bf49bfad382 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -58,7 +58,6 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # python-telegram-bot # requests certvalidator==0.11.1 @@ -150,8 +149,6 @@ iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest jaraco.classes==3.2.1 # via jaraco.collections jaraco.collections==3.4.0 @@ -228,8 +225,6 @@ ncclient==0.6.9 # via junos-eznc netaddr==0.7.19 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==22.0 @@ -281,9 +276,7 @@ pyiface==0.0.11 pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # adal - # twilio + # via twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -381,8 +374,6 @@ redis-py-cluster==2.1.3 # via -r requirements/static/ci/linux.in redis==3.5.3 # via redis-py-cluster -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt @@ -392,10 +383,8 @@ requests==2.25.1 # etcd3-py # kubernetes # moto - # msrest # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -427,7 +416,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kazoo diff --git a/requirements/static/ci/py3.11/freebsd.txt b/requirements/static/ci/py3.11/freebsd.txt index ae52c65407e..4d9264a3ec0 100644 --- a/requirements/static/ci/py3.11/freebsd.txt +++ b/requirements/static/ci/py3.11/freebsd.txt @@ -50,7 +50,6 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # requests certvalidator==0.11.1 # via vcert @@ -138,8 +137,6 @@ iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest jaraco.classes==3.2.1 # via jaraco.collections jaraco.collections==3.4.0 @@ -204,8 +201,6 @@ multidict==6.0.2 # via # aiohttp # yarl -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -249,8 +244,6 @@ pygit2==1.8.0 ; python_version >= "3.7" # via -r requirements/static/ci/freebsd.in pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in -pyjwt==2.4.0 - # via adal pynacl==1.3.0 # via paramiko pyopenssl==23.0.0 @@ -329,8 +322,6 @@ pyzmq==25.0.2 ; python_version >= "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt @@ -340,9 +331,7 @@ requests==2.25.1 # etcd3-py # kubernetes # moto - # msrest # pyvmomi - # requests-oauthlib # responses # vcert responses==0.10.6 @@ -366,7 +355,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # kazoo # kubernetes diff --git a/requirements/static/ci/py3.11/linux.txt b/requirements/static/ci/py3.11/linux.txt index 869451c0f6f..f6c008d55b1 100644 --- a/requirements/static/ci/py3.11/linux.txt +++ b/requirements/static/ci/py3.11/linux.txt @@ -58,7 +58,6 @@ certifi==2022.12.7 # via # -r requirements/static/ci/common.in # kubernetes - # msrest # python-telegram-bot # requests certvalidator==0.11.1 @@ -93,7 +92,6 @@ croniter==0.3.29 ; sys_platform != "win32" cryptography==39.0.2 # via # -r requirements/static/pkg/linux.in - # adal # ansible-core # etcd3-py # moto @@ -149,8 +147,6 @@ iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -isodate==0.6.0 - # via msrest jaraco.classes==3.2.1 # via jaraco.collections jaraco.collections==3.4.0 @@ -216,8 +212,6 @@ multidict==6.0.2 # via # aiohttp # yarl -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==22.0 @@ -265,9 +259,7 @@ pyiface==0.0.11 pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # adal - # twilio + # via twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -359,8 +351,6 @@ redis-py-cluster==2.1.3 # via -r requirements/static/ci/linux.in redis==3.5.3 # via redis-py-cluster -requests-oauthlib==1.3.0 - # via msrest requests==2.25.1 # via # -r requirements/base.txt @@ -370,10 +360,8 @@ requests==2.25.1 # etcd3-py # kubernetes # moto - # msrest # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -403,7 +391,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # kazoo # kubernetes diff --git a/requirements/static/ci/py3.7/cloud.txt b/requirements/static/ci/py3.7/cloud.txt index 2aefd51e45e..3a4218ca8bf 100644 --- a/requirements/static/ci/py3.7/cloud.txt +++ b/requirements/static/ci/py3.7/cloud.txt @@ -193,10 +193,14 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in +libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in loguru==0.6.0 # via ciscoconfparse looseversion==1.0.2 @@ -259,8 +263,9 @@ packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -296,6 +301,8 @@ pycryptodomex==3.10.1 # via -r requirements/crypto.txt pyeapi==0.8.4 # via napalm +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pynacl==1.4.0 # via paramiko pyopenssl==23.0.0 @@ -437,6 +444,7 @@ six==1.16.0 # geomet # jsonschema # junos-eznc + # kazoo # kubernetes # ncclient # paramiko diff --git a/requirements/static/ci/py3.7/docs.txt b/requirements/static/ci/py3.7/docs.txt index 7876517ba0e..6bc786d2c21 100644 --- a/requirements/static/ci/py3.7/docs.txt +++ b/requirements/static/ci/py3.7/docs.txt @@ -49,23 +49,16 @@ importlib-metadata==4.6.4 # -c requirements/static/ci/py3.7/linux.txt # sphinxcontrib-spelling jaraco.classes==3.2.1 - # via - # -c requirements/static/ci/py3.7/linux.txt - # jaraco.collections + # via jaraco.collections jaraco.collections==3.4.0 - # via - # -c requirements/static/ci/py3.7/linux.txt - # cherrypy + # via cherrypy jaraco.functools==2.0 # via - # -c requirements/static/ci/py3.7/linux.txt # cheroot # jaraco.text # tempora jaraco.text==3.5.1 - # via - # -c requirements/static/ci/py3.7/linux.txt - # jaraco.collections + # via jaraco.collections jinja2==3.1.2 # via # -c requirements/static/ci/py3.7/linux.txt @@ -198,9 +191,7 @@ urllib3==1.26.6 # -c requirements/static/ci/py3.7/linux.txt # requests zc.lockfile==1.4 - # via - # -c requirements/static/ci/py3.7/linux.txt - # cherrypy + # via cherrypy zipp==3.5.0 # via # -c requirements/static/ci/py3.7/linux.txt diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt index 778cdda0242..e915fe69d65 100644 --- a/requirements/static/ci/py3.7/freebsd.txt +++ b/requirements/static/ci/py3.7/freebsd.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: +# This file is autogenerated by pip-compile +# To update, run: # # pip-compile --output-file=requirements/static/ci/py3.7/freebsd.txt --pip-args='--constraint=requirements/static/pkg/py3.7/freebsd.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/freebsd.in requirements/static/pkg/freebsd.in requirements/zeromq.txt # @@ -16,6 +16,8 @@ asn1crypto==1.3.0 # oscrypto async-timeout==4.0.2 # via aiohttp +asynctest==0.13.0 + # via aiohttp attrs==20.3.0 # via # aiohttp @@ -25,18 +27,20 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -backports-entry-points-selectable==1.1.0 +autocommand==2.2.2 + # via jaraco.text +backports.entry-points-selectable==1.2.0 # via virtualenv bcrypt==3.1.6 # via # paramiko # passlib -boto==2.49.0 - # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto +boto==2.49.0 + # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -144,22 +148,34 @@ idna==2.8 immutables==0.15 # via contextvars importlib-metadata==4.6.4 - # via -r requirements/static/pkg/freebsd.in + # via + # -r requirements/static/pkg/freebsd.in + # backports.entry-points-selectable + # jsonschema + # mako + # moto + # pluggy + # pytest + # virtualenv +importlib-resources==5.12.0 + # via jaraco.text +inflect==6.0.4 + # via jaraco.text iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -jaraco-classes==3.2.1 - # via jaraco-collections -jaraco-collections==3.4.0 +jaraco.collections==4.1.0 # via cherrypy -jaraco-functools==2.0 +jaraco.context==4.3.0 + # via jaraco.text +jaraco.functools==3.6.0 # via # cheroot - # jaraco-text + # jaraco.text # tempora -jaraco-text==3.5.1 - # via jaraco-collections +jaraco.text==3.11.1 + # via jaraco.collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -180,10 +196,14 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in +libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -208,8 +228,8 @@ more-itertools==5.0.0 # via # cheroot # cherrypy - # jaraco-classes - # jaraco-functools + # jaraco.functools + # jaraco.text moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -239,8 +259,9 @@ packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -266,23 +287,24 @@ psutil==5.8.0 # pytest-salt-factories # pytest-shell-utilities # pytest-system-statistics +pyasn1-modules==0.2.4 + # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.4 - # via google-auth -pycparser==2.21 ; python_version >= "3.9" - # via - # -r requirements/static/ci/common.in - # -r requirements/static/pkg/freebsd.in - # cffi +pycparser==2.21 + # via cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt +pydantic==1.10.7 + # via inflect pyeapi==0.8.3 # via napalm pygit2==1.8.0 ; python_version >= "3.7" # via -r requirements/static/ci/freebsd.in +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pynacl==1.3.0 # via paramiko pyopenssl==23.0.0 @@ -299,18 +321,6 @@ pyserial==3.4 # via # junos-eznc # netmiko -pytest==7.2.0 ; python_version > "3.6" - # via - # -r requirements/pytest.txt - # pytest-custom-exit-code - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-subtests - # pytest-system-statistics - # pytest-tempdir - # pytest-timeout pytest-custom-exit-code==0.3.0 # via -r requirements/pytest.txt pytest-helpers-namespace==2021.4.29 @@ -339,6 +349,18 @@ pytest-tempdir==2019.10.12 # pytest-salt-factories pytest-timeout==1.4.2 # via -r requirements/pytest.txt +pytest==7.2.0 ; python_version > "3.6" + # via + # -r requirements/pytest.txt + # pytest-custom-exit-code + # pytest-helpers-namespace + # pytest-salt-factories + # pytest-shell-utilities + # pytest-skip-markers + # pytest-subtests + # pytest-system-statistics + # pytest-tempdir + # pytest-timeout python-dateutil==2.8.1 # via # -r requirements/static/pkg/freebsd.in @@ -411,6 +433,7 @@ six==1.16.0 # google-auth # jsonschema # junos-eznc + # kazoo # kubernetes # mock # more-itertools @@ -448,10 +471,16 @@ tomli==2.0.1 # via pytest transitions==0.8.1 # via junos-eznc -typing-extensions==3.10.0.0 +typing-extensions==4.5.0 # via + # aiohttp + # async-timeout + # gitpython + # importlib-metadata + # pydantic # pytest-shell-utilities # pytest-system-statistics + # yarl urllib3==1.26.6 # via # botocore @@ -482,10 +511,12 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc-lockfile==1.4 +zc.lockfile==3.0.post1 # via cherrypy zipp==3.5.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/ci/py3.8/cloud.txt b/requirements/static/ci/py3.8/cloud.txt index b5a3fb09393..e1450e08eb4 100644 --- a/requirements/static/ci/py3.8/cloud.txt +++ b/requirements/static/ci/py3.8/cloud.txt @@ -182,10 +182,14 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in +libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in loguru==0.6.0 # via ciscoconfparse looseversion==1.0.2 @@ -248,8 +252,9 @@ packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -285,6 +290,8 @@ pycryptodomex==3.10.1 # via -r requirements/crypto.txt pyeapi==0.8.4 # via napalm +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pynacl==1.4.0 # via paramiko pyopenssl==23.0.0 @@ -426,6 +433,7 @@ six==1.16.0 # geomet # jsonschema # junos-eznc + # kazoo # kubernetes # ncclient # paramiko diff --git a/requirements/static/ci/py3.8/docs.txt b/requirements/static/ci/py3.8/docs.txt index bd6095559a2..5a398c0e569 100644 --- a/requirements/static/ci/py3.8/docs.txt +++ b/requirements/static/ci/py3.8/docs.txt @@ -45,23 +45,16 @@ immutables==0.15 # -c requirements/static/ci/py3.8/linux.txt # contextvars jaraco.classes==3.2.1 - # via - # -c requirements/static/ci/py3.8/linux.txt - # jaraco.collections + # via jaraco.collections jaraco.collections==3.4.0 - # via - # -c requirements/static/ci/py3.8/linux.txt - # cherrypy + # via cherrypy jaraco.functools==2.0 # via - # -c requirements/static/ci/py3.8/linux.txt # cheroot # jaraco.text # tempora jaraco.text==3.5.1 - # via - # -c requirements/static/ci/py3.8/linux.txt - # jaraco.collections + # via jaraco.collections jinja2==3.1.2 # via # -c requirements/static/ci/py3.8/linux.txt @@ -188,9 +181,7 @@ urllib3==1.26.6 # -c requirements/static/ci/py3.8/linux.txt # requests zc.lockfile==1.4 - # via - # -c requirements/static/ci/py3.8/linux.txt - # cherrypy + # via cherrypy # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index 07a1ba0f7b7..86710f8db1a 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: +# This file is autogenerated by pip-compile +# To update, run: # # pip-compile --output-file=requirements/static/ci/py3.8/freebsd.txt --pip-args='--constraint=requirements/static/pkg/py3.8/freebsd.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/freebsd.in requirements/static/pkg/freebsd.in requirements/zeromq.txt # @@ -25,18 +25,20 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -backports-entry-points-selectable==1.1.0 +autocommand==2.2.2 + # via jaraco.text +backports.entry-points-selectable==1.2.0 # via virtualenv bcrypt==3.1.6 # via # paramiko # passlib -boto==2.49.0 - # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto +boto==2.49.0 + # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -145,21 +147,25 @@ immutables==0.15 # via contextvars importlib-metadata==4.6.4 # via -r requirements/static/pkg/freebsd.in +importlib-resources==5.12.0 + # via jaraco.text +inflect==6.0.4 + # via jaraco.text iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -jaraco-classes==3.2.1 - # via jaraco-collections -jaraco-collections==3.4.0 +jaraco.collections==4.1.0 # via cherrypy -jaraco-functools==2.0 +jaraco.context==4.3.0 + # via jaraco.text +jaraco.functools==3.6.0 # via # cheroot - # jaraco-text + # jaraco.text # tempora -jaraco-text==3.5.1 - # via jaraco-collections +jaraco.text==3.11.1 + # via jaraco.collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -180,10 +186,14 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in +libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -208,8 +218,8 @@ more-itertools==5.0.0 # via # cheroot # cherrypy - # jaraco-classes - # jaraco-functools + # jaraco.functools + # jaraco.text moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -239,8 +249,9 @@ packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -266,23 +277,24 @@ psutil==5.8.0 # pytest-salt-factories # pytest-shell-utilities # pytest-system-statistics +pyasn1-modules==0.2.4 + # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.4 - # via google-auth -pycparser==2.21 ; python_version >= "3.9" - # via - # -r requirements/static/ci/common.in - # -r requirements/static/pkg/freebsd.in - # cffi +pycparser==2.21 + # via cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt +pydantic==1.10.7 + # via inflect pyeapi==0.8.3 # via napalm pygit2==1.8.0 ; python_version >= "3.7" # via -r requirements/static/ci/freebsd.in +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pynacl==1.3.0 # via paramiko pyopenssl==23.0.0 @@ -299,18 +311,6 @@ pyserial==3.4 # via # junos-eznc # netmiko -pytest==7.2.0 ; python_version > "3.6" - # via - # -r requirements/pytest.txt - # pytest-custom-exit-code - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-subtests - # pytest-system-statistics - # pytest-tempdir - # pytest-timeout pytest-custom-exit-code==0.3.0 # via -r requirements/pytest.txt pytest-helpers-namespace==2021.4.29 @@ -339,6 +339,18 @@ pytest-tempdir==2019.10.12 # pytest-salt-factories pytest-timeout==1.4.2 # via -r requirements/pytest.txt +pytest==7.2.0 ; python_version > "3.6" + # via + # -r requirements/pytest.txt + # pytest-custom-exit-code + # pytest-helpers-namespace + # pytest-salt-factories + # pytest-shell-utilities + # pytest-skip-markers + # pytest-subtests + # pytest-system-statistics + # pytest-tempdir + # pytest-timeout python-dateutil==2.8.1 # via # -r requirements/static/pkg/freebsd.in @@ -411,6 +423,7 @@ six==1.16.0 # google-auth # jsonschema # junos-eznc + # kazoo # kubernetes # mock # more-itertools @@ -450,6 +463,7 @@ transitions==0.8.1 # via junos-eznc typing-extensions==4.2.0 # via + # pydantic # pytest-shell-utilities # pytest-system-statistics urllib3==1.26.6 @@ -482,10 +496,12 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc-lockfile==1.4 +zc.lockfile==3.0.post1 # via cherrypy zipp==3.5.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/ci/py3.9/cloud.txt b/requirements/static/ci/py3.9/cloud.txt index f283439bf82..4ccd49bcdee 100644 --- a/requirements/static/ci/py3.9/cloud.txt +++ b/requirements/static/ci/py3.9/cloud.txt @@ -182,10 +182,14 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in +libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in loguru==0.6.0 # via ciscoconfparse looseversion==1.0.2 @@ -248,8 +252,9 @@ packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -288,6 +293,8 @@ pycryptodomex==3.10.1 # via -r requirements/crypto.txt pyeapi==0.8.4 # via napalm +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pynacl==1.4.0 # via paramiko pyopenssl==23.0.0 @@ -429,6 +436,7 @@ six==1.16.0 # geomet # jsonschema # junos-eznc + # kazoo # kubernetes # ncclient # paramiko diff --git a/requirements/static/ci/py3.9/docs.txt b/requirements/static/ci/py3.9/docs.txt index 3067f1725f7..c40924e4b63 100644 --- a/requirements/static/ci/py3.9/docs.txt +++ b/requirements/static/ci/py3.9/docs.txt @@ -6,6 +6,10 @@ # alabaster==0.7.12 # via sphinx +autocommand==2.2.2 + # via + # -c requirements/static/ci/py3.9/linux.txt + # jaraco.text babel==2.9.1 # via sphinx certifi==2022.12.7 @@ -48,21 +52,25 @@ importlib-metadata==6.0.0 # via # -c requirements/static/ci/py3.9/linux.txt # sphinx -jaraco.classes==3.2.1 +inflect==6.0.4 # via # -c requirements/static/ci/py3.9/linux.txt - # jaraco.collections -jaraco.collections==3.4.0 + # jaraco.text +jaraco.collections==4.1.0 # via # -c requirements/static/ci/py3.9/linux.txt # cherrypy -jaraco.functools==2.0 +jaraco.context==4.3.0 + # via + # -c requirements/static/ci/py3.9/linux.txt + # jaraco.text +jaraco.functools==3.6.0 # via # -c requirements/static/ci/py3.9/linux.txt # cheroot # jaraco.text # tempora -jaraco.text==3.5.1 +jaraco.text==3.11.1 # via # -c requirements/static/ci/py3.9/linux.txt # jaraco.collections @@ -100,8 +108,8 @@ more-itertools==5.0.0 # -c requirements/static/ci/py3.9/linux.txt # cheroot # cherrypy - # jaraco.classes # jaraco.functools + # jaraco.text msgpack==1.0.2 # via # -c requirements/static/ci/py3.9/linux.txt @@ -125,6 +133,10 @@ pycryptodomex==3.9.8 # via # -c requirements/static/ci/py3.9/linux.txt # -r requirements/crypto.txt +pydantic==1.10.7 + # via + # -c requirements/static/ci/py3.9/linux.txt + # inflect pyenchant==3.2.2 # via sphinxcontrib-spelling pygments==2.14.0 @@ -185,13 +197,14 @@ typing-extensions==4.2.0 # via # -c requirements/static/ci/py3.9/linux.txt # myst-docutils + # pydantic uc-micro-py==1.0.1 # via linkify-it-py urllib3==1.26.6 # via # -c requirements/static/ci/py3.9/linux.txt # requests -zc.lockfile==1.4 +zc.lockfile==3.0.post1 # via # -c requirements/static/ci/py3.9/linux.txt # cherrypy diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index f24fc5d81bb..f845df80c58 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: +# This file is autogenerated by pip-compile +# To update, run: # # pip-compile --output-file=requirements/static/ci/py3.9/freebsd.txt --pip-args='--constraint=requirements/static/pkg/py3.9/freebsd.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/freebsd.in requirements/static/pkg/freebsd.in requirements/zeromq.txt # @@ -25,18 +25,20 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -backports-entry-points-selectable==1.1.0 +autocommand==2.2.2 + # via jaraco.text +backports.entry-points-selectable==1.2.0 # via virtualenv bcrypt==3.1.6 # via # paramiko # passlib -boto==2.49.0 - # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto +boto==2.49.0 + # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -145,21 +147,23 @@ immutables==0.15 # via contextvars importlib-metadata==6.0.0 # via -r requirements/static/pkg/freebsd.in +inflect==6.0.4 + # via jaraco.text iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -jaraco-classes==3.2.1 - # via jaraco-collections -jaraco-collections==3.4.0 +jaraco.collections==4.1.0 # via cherrypy -jaraco-functools==2.0 +jaraco.context==4.3.0 + # via jaraco.text +jaraco.functools==3.6.0 # via # cheroot - # jaraco-text + # jaraco.text # tempora -jaraco-text==3.5.1 - # via jaraco-collections +jaraco.text==3.11.1 + # via jaraco.collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -180,10 +184,14 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in +libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -208,8 +216,8 @@ more-itertools==5.0.0 # via # cheroot # cherrypy - # jaraco-classes - # jaraco-functools + # jaraco.functools + # jaraco.text moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -239,8 +247,9 @@ packaging==21.3 # via # -r requirements/base.txt # pytest -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -266,12 +275,12 @@ psutil==5.8.0 # pytest-salt-factories # pytest-shell-utilities # pytest-system-statistics +pyasn1-modules==0.2.4 + # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.4 - # via google-auth pycparser==2.21 ; python_version >= "3.9" # via # -r requirements/static/ci/common.in @@ -279,10 +288,14 @@ pycparser==2.21 ; python_version >= "3.9" # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt +pydantic==1.10.7 + # via inflect pyeapi==0.8.3 # via napalm pygit2==1.8.0 ; python_version >= "3.7" # via -r requirements/static/ci/freebsd.in +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pynacl==1.3.0 # via paramiko pyopenssl==23.0.0 @@ -299,18 +312,6 @@ pyserial==3.4 # via # junos-eznc # netmiko -pytest==7.2.0 ; python_version > "3.6" - # via - # -r requirements/pytest.txt - # pytest-custom-exit-code - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-subtests - # pytest-system-statistics - # pytest-tempdir - # pytest-timeout pytest-custom-exit-code==0.3.0 # via -r requirements/pytest.txt pytest-helpers-namespace==2021.4.29 @@ -339,6 +340,18 @@ pytest-tempdir==2019.10.12 # pytest-salt-factories pytest-timeout==1.4.2 # via -r requirements/pytest.txt +pytest==7.2.0 ; python_version > "3.6" + # via + # -r requirements/pytest.txt + # pytest-custom-exit-code + # pytest-helpers-namespace + # pytest-salt-factories + # pytest-shell-utilities + # pytest-skip-markers + # pytest-subtests + # pytest-system-statistics + # pytest-tempdir + # pytest-timeout python-dateutil==2.8.1 # via # -r requirements/static/pkg/freebsd.in @@ -411,6 +424,7 @@ six==1.16.0 # google-auth # jsonschema # junos-eznc + # kazoo # kubernetes # mock # more-itertools @@ -450,6 +464,7 @@ transitions==0.8.1 # via junos-eznc typing-extensions==4.2.0 # via + # pydantic # pytest-shell-utilities # pytest-system-statistics urllib3==1.26.6 @@ -482,7 +497,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc-lockfile==1.4 +zc.lockfile==3.0.post1 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.9/lint.txt b/requirements/static/ci/py3.9/lint.txt index e2a83aa2d99..02529e9a73d 100644 --- a/requirements/static/ci/py3.9/lint.txt +++ b/requirements/static/ci/py3.9/lint.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: +# This file is autogenerated by pip-compile +# To update, run: # # pip-compile --output-file=requirements/static/ci/py3.9/lint.txt --pip-args='--constraint=requirements/static/ci/py3.9/linux.txt' requirements/base.txt requirements/static/ci/common.in requirements/static/ci/lint.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # @@ -8,10 +8,10 @@ aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 # via aiohttp -ansible==7.1.0 ; python_version >= "3.9" - # via -r requirements/static/ci/linux.in ansible-core==2.14.1 # via ansible +ansible==7.1.0 ; python_version >= "3.9" + # via -r requirements/static/ci/linux.in apache-libcloud==3.3.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in apscheduler==3.6.3 @@ -28,18 +28,20 @@ attrs==21.2.0 # via # aiohttp # jsonschema -backports-entry-points-selectable==1.1.0 +autocommand==2.2.2 + # via jaraco.text +backports.entry-points-selectable==1.2.0 # via virtualenv bcrypt==3.2.0 # via # paramiko # passlib -boto==2.49.0 - # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto +boto==2.49.0 + # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -144,21 +146,23 @@ immutables==0.16 # via contextvars importlib-metadata==6.0.0 # via -r requirements/static/pkg/linux.in +inflect==6.0.4 + # via jaraco.text ipaddress==1.0.23 # via kubernetes isort==4.3.21 # via pylint -jaraco-classes==3.2.1 - # via jaraco-collections -jaraco-collections==3.4.0 +jaraco.collections==4.1.0 # via cherrypy -jaraco-functools==3.3.0 +jaraco.context==4.3.0 + # via jaraco.text +jaraco.functools==3.6.0 # via # cheroot - # jaraco-text + # jaraco.text # tempora -jaraco-text==3.5.1 - # via jaraco-collections +jaraco.text==3.11.1 + # via jaraco.collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -180,12 +184,16 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in lazy-object-proxy==1.4.3 # via astroid +libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in loguru==0.6.0 # via ciscoconfparse looseversion==1.0.2 @@ -214,8 +222,8 @@ more-itertools==8.8.0 # via # cheroot # cherrypy - # jaraco-classes - # jaraco-functools + # jaraco.functools + # jaraco.text moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -247,8 +255,9 @@ packaging==21.3 # via # -r requirements/base.txt # ansible-core -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -266,12 +275,12 @@ portend==2.7.1 # via cherrypy psutil==5.8.0 # via -r requirements/base.txt +pyasn1-modules==0.2.8 + # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.8 - # via google-auth pycodestyle==2.5.0 # via saltpylint pycparser==2.21 ; python_version >= "3.9" @@ -281,12 +290,16 @@ pycparser==2.21 ; python_version >= "3.9" # cffi pycryptodomex==3.10.1 # via -r requirements/crypto.txt +pydantic==1.10.7 + # via inflect pyeapi==0.8.4 # via napalm pygit2==1.6.1 ; python_version > "3.8" # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pyjwt==2.4.0 # via twilio pylint==2.4.4 @@ -348,10 +361,10 @@ pyyaml==5.4.1 # yamlordereddictloader pyzmq==23.2.0 ; python_version < "3.11" # via -r requirements/zeromq.txt -redis==3.5.3 - # via redis-py-cluster redis-py-cluster==2.1.3 # via -r requirements/static/ci/linux.in +redis==3.5.3 + # via redis-py-cluster requests==2.26.0 # via # -r requirements/base.txt @@ -402,6 +415,7 @@ six==1.16.0 # geomet # jsonschema # junos-eznc + # kazoo # kubernetes # ncclient # paramiko @@ -445,6 +459,8 @@ transitions==0.8.8 # via junos-eznc twilio==7.9.2 # via -r requirements/static/ci/linux.in +typing-extensions==4.5.0 + # via pydantic tzlocal==3.0 # via apscheduler urllib3==1.26.6 @@ -476,7 +492,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc-lockfile==2.0 +zc.lockfile==3.0.post1 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index e5fbc53e96e..f255b3348a5 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: +# This file is autogenerated by pip-compile +# To update, run: # # pip-compile --output-file=requirements/static/ci/py3.9/linux.txt --pip-args='--constraint=requirements/static/pkg/py3.9/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # @@ -8,10 +8,10 @@ aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 # via aiohttp -ansible==7.1.0 ; python_version >= "3.9" - # via -r requirements/static/ci/linux.in ansible-core==2.14.1 # via ansible +ansible==7.1.0 ; python_version >= "3.9" + # via -r requirements/static/ci/linux.in apache-libcloud==2.5.0 ; sys_platform != "win32" # via -r requirements/static/ci/common.in apscheduler==3.6.3 @@ -31,18 +31,20 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -backports-entry-points-selectable==1.1.0 +autocommand==2.2.2 + # via jaraco.text +backports.entry-points-selectable==1.2.0 # via virtualenv bcrypt==3.1.6 # via # paramiko # passlib -boto==2.49.0 - # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto +boto==2.49.0 + # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -157,21 +159,23 @@ immutables==0.15 # via contextvars importlib-metadata==6.0.0 # via -r requirements/static/pkg/linux.in +inflect==6.0.4 + # via jaraco.text iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -jaraco-classes==3.2.1 - # via jaraco-collections -jaraco-collections==3.4.0 +jaraco.collections==4.1.0 # via cherrypy -jaraco-functools==2.0 +jaraco.context==4.3.0 + # via jaraco.text +jaraco.functools==3.6.0 # via # cheroot - # jaraco-text + # jaraco.text # tempora -jaraco-text==3.5.1 - # via jaraco-collections +jaraco.text==3.11.1 + # via jaraco.collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -193,10 +197,14 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in +libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -221,8 +229,8 @@ more-itertools==5.0.0 # via # cheroot # cherrypy - # jaraco-classes - # jaraco-functools + # jaraco.functools + # jaraco.text moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -253,8 +261,9 @@ packaging==22.0 # -r requirements/base.txt # ansible-core # pytest -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -280,12 +289,12 @@ psutil==5.8.0 # pytest-salt-factories # pytest-shell-utilities # pytest-system-statistics +pyasn1-modules==0.2.4 + # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.4 - # via google-auth pycparser==2.21 ; python_version >= "3.9" # via # -r requirements/static/ci/common.in @@ -293,12 +302,16 @@ pycparser==2.21 ; python_version >= "3.9" # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt +pydantic==1.10.7 + # via inflect pyeapi==0.8.3 # via napalm pygit2==1.5.0 ; python_version > "3.8" # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pyjwt==2.4.0 # via twilio pymysql==1.0.2 ; python_version > "3.5" @@ -317,18 +330,6 @@ pyserial==3.4 # via # junos-eznc # netmiko -pytest==7.2.0 ; python_version > "3.6" - # via - # -r requirements/pytest.txt - # pytest-custom-exit-code - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-subtests - # pytest-system-statistics - # pytest-tempdir - # pytest-timeout pytest-custom-exit-code==0.3.0 # via -r requirements/pytest.txt pytest-helpers-namespace==2021.4.29 @@ -357,6 +358,18 @@ pytest-tempdir==2019.10.12 # pytest-salt-factories pytest-timeout==1.4.2 # via -r requirements/pytest.txt +pytest==7.2.0 ; python_version > "3.6" + # via + # -r requirements/pytest.txt + # pytest-custom-exit-code + # pytest-helpers-namespace + # pytest-salt-factories + # pytest-shell-utilities + # pytest-skip-markers + # pytest-subtests + # pytest-system-statistics + # pytest-tempdir + # pytest-timeout python-consul==1.1.0 # via -r requirements/static/ci/linux.in python-dateutil==2.8.1 @@ -397,10 +410,10 @@ pyzmq==23.2.0 ; python_version < "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -redis==3.5.3 - # via redis-py-cluster redis-py-cluster==2.1.3 # via -r requirements/static/ci/linux.in +redis==3.5.3 + # via redis-py-cluster requests==2.25.1 # via # -r requirements/base.txt @@ -449,6 +462,7 @@ six==1.16.0 # google-auth # jsonschema # junos-eznc + # kazoo # kubernetes # mock # more-itertools @@ -497,6 +511,7 @@ twilio==7.9.2 # via -r requirements/static/ci/linux.in typing-extensions==4.2.0 # via + # pydantic # pytest-shell-utilities # pytest-system-statistics tzlocal==2.1 @@ -531,7 +546,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc-lockfile==1.4 +zc.lockfile==3.0.post1 # via cherrypy zipp==3.5.0 # via importlib-metadata From 7247f37e0ff9f932d6382c1d3792138967de3162 Mon Sep 17 00:00:00 2001 From: natalieswork Date: Mon, 22 May 2023 16:50:28 -0400 Subject: [PATCH 045/152] fixes saltstack/salt#64322 adding change log --- changelog/64322.removed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/64322.removed.md diff --git a/changelog/64322.removed.md b/changelog/64322.removed.md new file mode 100644 index 00000000000..fe7916f991f --- /dev/null +++ b/changelog/64322.removed.md @@ -0,0 +1 @@ +Removing Azure-Cloud modules from the code base. From d933bec9892c6cfc729218d3b11a4e1d169e385f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 22 May 2023 11:34:49 +0100 Subject: [PATCH 046/152] Try harder to detect what is the target release for changelog generation. Signed-off-by: Pedro Algarvio --- .github/workflows/ci.yml | 10 +++- .github/workflows/nightly.yml | 10 +++- .github/workflows/scheduled.yml | 10 +++- .github/workflows/staging.yml | 10 +++- .github/workflows/templates/ci.yml.jinja | 4 +- .github/workflows/templates/layout.yml.jinja | 7 +++ cicd/shared-gh-workflows-context.yml | 2 + tools/ci.py | 56 ++++++++++++++++++++ 8 files changed, 99 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4afb4f49df..a286234889a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }} latest-release: ${{ steps.get-salt-releases.outputs.latest-release }} releases: ${{ steps.get-salt-releases.outputs.releases }} + release-changelog-target: ${{ steps.get-release-changelog-target.outputs.release-changelog-target }} steps: - uses: actions/checkout@v3 with: @@ -243,6 +244,11 @@ jobs: id: set-cache-seed run: | echo "cache-seed=${{ env.CACHE_SEED }}" >> "$GITHUB_OUTPUT" + + - name: Get Release Changelog Target + id: get-release-changelog-target + run: | + tools ci get-release-changelog-target ${{ github.event_name }} pre-commit: name: Pre-Commit if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} @@ -320,7 +326,7 @@ jobs: shell: bash if: ${{ startsWith(github.event.ref, 'refs/tags') == false }} run: | - if [ "${{ github.base_ref || github.ref_name }}" == "master" ]; then + if [ "${{ needs.prepare-workflow.outputs.release-changelog-target }}" == "next-major-release" ]; then tools changelog update-release-notes --next-release --template-only else tools changelog update-release-notes --template-only @@ -330,7 +336,7 @@ jobs: shell: bash if: ${{ startsWith(github.event.ref, 'refs/tags') == false }} run: | - if [ "${{ github.base_ref || github.ref_name }}" == "master" ]; then + if [ "${{ needs.prepare-workflow.outputs.release-changelog-target }}" == "next-major-release" ]; then tools changelog update-release-notes --draft --next-release tools changelog update-release-notes --next-release else diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 69edd3bc2ae..1bdcfe65086 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -92,6 +92,7 @@ jobs: cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }} latest-release: ${{ steps.get-salt-releases.outputs.latest-release }} releases: ${{ steps.get-salt-releases.outputs.releases }} + release-changelog-target: ${{ steps.get-release-changelog-target.outputs.release-changelog-target }} steps: - uses: actions/checkout@v3 with: @@ -296,6 +297,11 @@ jobs: id: set-cache-seed run: | echo "cache-seed=${{ env.CACHE_SEED }}" >> "$GITHUB_OUTPUT" + + - name: Get Release Changelog Target + id: get-release-changelog-target + run: | + tools ci get-release-changelog-target ${{ github.event_name }} pre-commit: name: Pre-Commit if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} @@ -373,7 +379,7 @@ jobs: shell: bash if: ${{ startsWith(github.event.ref, 'refs/tags') == false }} run: | - if [ "${{ github.base_ref || github.ref_name }}" == "master" ]; then + if [ "${{ needs.prepare-workflow.outputs.release-changelog-target }}" == "next-major-release" ]; then tools changelog update-release-notes --next-release --template-only else tools changelog update-release-notes --template-only @@ -383,7 +389,7 @@ jobs: shell: bash if: ${{ startsWith(github.event.ref, 'refs/tags') == false }} run: | - if [ "${{ github.base_ref || github.ref_name }}" == "master" ]; then + if [ "${{ needs.prepare-workflow.outputs.release-changelog-target }}" == "next-major-release" ]; then tools changelog update-release-notes --draft --next-release tools changelog update-release-notes --next-release else diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 7e45d9a095b..368089af2df 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -82,6 +82,7 @@ jobs: cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }} latest-release: ${{ steps.get-salt-releases.outputs.latest-release }} releases: ${{ steps.get-salt-releases.outputs.releases }} + release-changelog-target: ${{ steps.get-release-changelog-target.outputs.release-changelog-target }} steps: - uses: actions/checkout@v3 with: @@ -286,6 +287,11 @@ jobs: id: set-cache-seed run: | echo "cache-seed=${{ env.CACHE_SEED }}" >> "$GITHUB_OUTPUT" + + - name: Get Release Changelog Target + id: get-release-changelog-target + run: | + tools ci get-release-changelog-target ${{ github.event_name }} pre-commit: name: Pre-Commit if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} @@ -363,7 +369,7 @@ jobs: shell: bash if: ${{ startsWith(github.event.ref, 'refs/tags') == false }} run: | - if [ "${{ github.base_ref || github.ref_name }}" == "master" ]; then + if [ "${{ needs.prepare-workflow.outputs.release-changelog-target }}" == "next-major-release" ]; then tools changelog update-release-notes --next-release --template-only else tools changelog update-release-notes --template-only @@ -373,7 +379,7 @@ jobs: shell: bash if: ${{ startsWith(github.event.ref, 'refs/tags') == false }} run: | - if [ "${{ github.base_ref || github.ref_name }}" == "master" ]; then + if [ "${{ needs.prepare-workflow.outputs.release-changelog-target }}" == "next-major-release" ]; then tools changelog update-release-notes --draft --next-release tools changelog update-release-notes --next-release else diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index dba7145bc1e..424f47c363e 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -72,6 +72,7 @@ jobs: cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }} latest-release: ${{ steps.get-salt-releases.outputs.latest-release }} releases: ${{ steps.get-salt-releases.outputs.releases }} + release-changelog-target: ${{ steps.get-release-changelog-target.outputs.release-changelog-target }} steps: - uses: actions/checkout@v3 with: @@ -282,6 +283,11 @@ jobs: id: set-cache-seed run: | echo "cache-seed=${{ env.CACHE_SEED }}" >> "$GITHUB_OUTPUT" + + - name: Get Release Changelog Target + id: get-release-changelog-target + run: | + tools ci get-release-changelog-target ${{ github.event_name }} pre-commit: name: Pre-Commit if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} @@ -363,7 +369,7 @@ jobs: shell: bash if: ${{ startsWith(github.event.ref, 'refs/tags') == false }} run: | - if [ "${{ github.base_ref || github.ref_name }}" == "master" ]; then + if [ "${{ needs.prepare-workflow.outputs.release-changelog-target }}" == "next-major-release" ]; then tools changelog update-release-notes --next-release --template-only else tools changelog update-release-notes --template-only @@ -373,7 +379,7 @@ jobs: shell: bash if: ${{ startsWith(github.event.ref, 'refs/tags') == false }} run: | - if [ "${{ github.base_ref || github.ref_name }}" == "master" ]; then + if [ "${{ needs.prepare-workflow.outputs.release-changelog-target }}" == "next-major-release" ]; then tools changelog update-release-notes --draft --release --next-release tools changelog update-release-notes --release --next-release else diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 080967fa583..2ed95a9218f 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -116,7 +116,7 @@ on: shell: bash if: ${{ startsWith(github.event.ref, 'refs/tags') == false }} run: | - if [ "${{ github.base_ref || github.ref_name }}" == "master" ]; then + if [ "${{ needs.prepare-workflow.outputs.release-changelog-target }}" == "next-major-release" ]; then tools changelog update-release-notes --next-release --template-only else tools changelog update-release-notes --template-only @@ -126,7 +126,7 @@ on: shell: bash if: ${{ startsWith(github.event.ref, 'refs/tags') == false }} run: | - if [ "${{ github.base_ref || github.ref_name }}" == "master" ]; then + if [ "${{ needs.prepare-workflow.outputs.release-changelog-target }}" == "next-major-release" ]; then tools changelog update-release-notes --draft <%- if prepare_actual_release %> --release <%- endif %> --next-release tools changelog update-release-notes <%- if prepare_actual_release %> --release <%- endif %> --next-release else diff --git a/.github/workflows/templates/layout.yml.jinja b/.github/workflows/templates/layout.yml.jinja index 59c2493b485..8fa64b89ad9 100644 --- a/.github/workflows/templates/layout.yml.jinja +++ b/.github/workflows/templates/layout.yml.jinja @@ -91,6 +91,7 @@ jobs: cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }} latest-release: ${{ steps.get-salt-releases.outputs.latest-release }} releases: ${{ steps.get-salt-releases.outputs.releases }} + release-changelog-target: ${{ steps.get-release-changelog-target.outputs.release-changelog-target }} steps: - uses: actions/checkout@v3 with: @@ -305,6 +306,12 @@ jobs: id: set-cache-seed run: | echo "cache-seed=${{ env.CACHE_SEED }}" >> "$GITHUB_OUTPUT" + + - name: Get Release Changelog Target + id: get-release-changelog-target + run: | + tools ci get-release-changelog-target ${{ github.event_name }} + <%- endblock prepare_workflow_job %> <%- endif %> diff --git a/cicd/shared-gh-workflows-context.yml b/cicd/shared-gh-workflows-context.yml index ec3d939fe03..f304a534af8 100644 --- a/cicd/shared-gh-workflows-context.yml +++ b/cicd/shared-gh-workflows-context.yml @@ -2,3 +2,5 @@ python_version_linux: "3.10.11" python_version_macos: "3.10.11" python_version_windows: "3.10.11" relenv_version: "0.12.3" +release-branches: + - "3006.x" diff --git a/tools/ci.py b/tools/ci.py index ba7a7c2f849..db83c4e776f 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -11,6 +11,7 @@ import pathlib import time from typing import TYPE_CHECKING +import yaml from ptscripts import Context, command_group import tools.utils @@ -672,3 +673,58 @@ def get_releases(ctx: Context, repository: str = "saltstack/salt"): wfh.write(f"latest-release={latest}\n") wfh.write(f"releases={json.dumps(str_releases)}\n") ctx.exit(0) + + +@ci.command( + name="get-release-changelog-target", + arguments={ + "event_name": { + "help": "The name of the GitHub event being processed.", + }, + }, +) +def get_release_changelog_target(ctx: Context, event_name: str): + """ + Define which kind of release notes should be generated, next minor or major. + """ + gh_event_path = os.environ.get("GITHUB_EVENT_PATH") or None + if gh_event_path is None: + ctx.warn("The 'GITHUB_EVENT_PATH' variable is not set.") + ctx.exit(1) + + if TYPE_CHECKING: + assert gh_event_path is not None + + try: + gh_event = json.loads(open(gh_event_path).read()) + except Exception as exc: + ctx.error(f"Could not load the GH Event payload from {gh_event_path!r}:\n", exc) + ctx.exit(1) + + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output is None: + ctx.warn("The 'GITHUB_OUTPUT' variable is not set.") + ctx.exit(1) + + if TYPE_CHECKING: + assert github_output is not None + + shared_context_file = ( + tools.utils.REPO_ROOT / "cicd" / "shared-gh-workflows-context.yml" + ) + shared_context = yaml.safe_load(shared_context_file.read_text()) + release_branches = shared_context["release-branches"] + + release_changelog_target = "next-major-release" + if event_name == "pull_request": + if gh_event["pull_request"]["base"]["ref"] in release_branches: + release_changelog_target = "next-minor-release" + + else: + for branch_name in release_branches: + if branch_name in gh_event["ref"]: + release_changelog_target = "next-minor-release" + break + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"release-changelog-target={release_changelog_target}\n") + ctx.exit(0) From a81f58f37d21861567dc3d7feb22eaca2c03efd0 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 22 May 2023 11:41:06 +0100 Subject: [PATCH 047/152] Define a variable for the path to `shared-gh-workflows-context.yml` Signed-off-by: Pedro Algarvio --- tools/ci.py | 5 ++--- tools/pkg/build.py | 5 +---- tools/pre_commit.py | 5 ++--- tools/utils.py | 3 +++ 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tools/ci.py b/tools/ci.py index db83c4e776f..a554d798ec5 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -709,10 +709,9 @@ def get_release_changelog_target(ctx: Context, event_name: str): if TYPE_CHECKING: assert github_output is not None - shared_context_file = ( - tools.utils.REPO_ROOT / "cicd" / "shared-gh-workflows-context.yml" + shared_context = yaml.safe_load( + tools.utils.SHARED_WORKFLOW_CONTEXT_FILEPATH.read_text() ) - shared_context = yaml.safe_load(shared_context_file.read_text()) release_branches = shared_context["release-branches"] release_changelog_target = "next-major-release" diff --git a/tools/pkg/build.py b/tools/pkg/build.py index b373338a99e..8a99ba5eca4 100644 --- a/tools/pkg/build.py +++ b/tools/pkg/build.py @@ -30,10 +30,7 @@ build = command_group( def _get_shared_constants(): - shared_constants = ( - tools.utils.REPO_ROOT / "cicd" / "shared-gh-workflows-context.yml" - ) - return yaml.safe_load(shared_constants.read_text()) + return yaml.safe_load(tools.utils.SHARED_WORKFLOW_CONTEXT_FILEPATH.read_text()) @build.command( diff --git a/tools/pre_commit.py b/tools/pre_commit.py index af054876d80..c272d26821f 100644 --- a/tools/pre_commit.py +++ b/tools/pre_commit.py @@ -116,10 +116,9 @@ def generate_workflows(ctx: Context): "prepare_workflow_needs": NeedsTracker(), "build_repo_needs": NeedsTracker(), } - shared_context_file = ( - tools.utils.REPO_ROOT / "cicd" / "shared-gh-workflows-context.yml" + shared_context = yaml.safe_load( + tools.utils.SHARED_WORKFLOW_CONTEXT_FILEPATH.read_text() ) - shared_context = yaml.safe_load(shared_context_file.read_text()) for key, value in shared_context.items(): context[key] = value loaded_template = env.get_template(template_path.name) diff --git a/tools/utils.py b/tools/utils.py index 28a79745844..8369d25eafe 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -38,6 +38,9 @@ SPB_ENVIRONMENT = os.environ.get("SPB_ENVIRONMENT") or "prod" STAGING_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-staging" RELEASE_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-release" BACKUP_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-backup" +SHARED_WORKFLOW_CONTEXT_FILEPATH = ( + REPO_ROOT / "cicd" / "shared-gh-workflows-context.yml" +) class UpdateProgress: From f3edefc93a1f5c5ef927a2e97f9461b591701e28 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 22 May 2023 11:05:12 +0100 Subject: [PATCH 048/152] Update to latest golden images Signed-off-by: Pedro Algarvio --- cicd/amis.yml | 2 +- cicd/golden-images.json | 132 ++++++++++++++++++++-------------------- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/cicd/amis.yml b/cicd/amis.yml index 47edcf0184b..8fb4513180f 100644 --- a/cicd/amis.yml +++ b/cicd/amis.yml @@ -1 +1 @@ -centosstream-9-x86_64: ami-044545f7a74d46acc +centosstream-9-x86_64: ami-0bd92f4dca5d74017 diff --git a/cicd/golden-images.json b/cicd/golden-images.json index 75341e64aeb..02c3ee0977c 100644 --- a/cicd/golden-images.json +++ b/cicd/golden-images.json @@ -1,8 +1,8 @@ { "almalinux-8-arm64": { - "ami": "ami-0fc1e14bf9ff422aa", + "ami": "ami-05c1d3dbdeeb94bc6", "ami_description": "CI Image of AlmaLinux 8 arm64", - "ami_name": "salt-project/ci/almalinux/8/arm64/20230418.1731", + "ami_name": "salt-project/ci/almalinux/8/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -10,9 +10,9 @@ "ssh_username": "ec2-user" }, "almalinux-8": { - "ami": "ami-0bae4158c1f126271", + "ami": "ami-0ec1cbc531f10105b", "ami_description": "CI Image of AlmaLinux 8 x86_64", - "ami_name": "salt-project/ci/almalinux/8/x86_64/20230418.1732", + "ami_name": "salt-project/ci/almalinux/8/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -20,9 +20,9 @@ "ssh_username": "ec2-user" }, "almalinux-9-arm64": { - "ami": "ami-08f4d0fbf5d53c3ab", + "ami": "ami-036c495af9dfcf852", "ami_description": "CI Image of AlmaLinux 9 arm64", - "ami_name": "salt-project/ci/almalinux/9/arm64/20230418.1732", + "ami_name": "salt-project/ci/almalinux/9/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -30,9 +30,9 @@ "ssh_username": "ec2-user" }, "almalinux-9": { - "ami": "ami-00404c1cc5c5a08bd", + "ami": "ami-0dbc7030666419671", "ami_description": "CI Image of AlmaLinux 9 x86_64", - "ami_name": "salt-project/ci/almalinux/9/x86_64/20230418.1738", + "ami_name": "salt-project/ci/almalinux/9/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -40,9 +40,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2-arm64": { - "ami": "ami-05fbdb644d06c27b6", + "ami": "ami-022232915c2a5f2d0", "ami_description": "CI Image of AmazonLinux 2 arm64", - "ami_name": "salt-project/ci/amazonlinux/2/arm64/20230418.1717", + "ami_name": "salt-project/ci/amazonlinux/2/arm64/20230522.0621", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -50,9 +50,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2": { - "ami": "ami-014171e6c30ec8387", + "ami": "ami-0695f87baa5b5ce15", "ami_description": "CI Image of AmazonLinux 2 x86_64", - "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20230418.1718", + "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20230522.0620", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -60,9 +60,9 @@ "ssh_username": "ec2-user" }, "archlinux-lts": { - "ami": "ami-00cff81ed2e2fb0f4", + "ami": "ami-0f6424847f98afc04", "ami_description": "CI Image of ArchLinux lts x86_64", - "ami_name": "salt-project/ci/archlinux/lts/x86_64/20230418.1717", + "ami_name": "salt-project/ci/archlinux/lts/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "false", "instance_type": "t3a.large", @@ -70,9 +70,9 @@ "ssh_username": "arch" }, "centos-7-arm64": { - "ami": "ami-051cef43c13fcc0c9", + "ami": "ami-0908831c364e33a37", "ami_description": "CI Image of CentOS 7 arm64", - "ami_name": "salt-project/ci/centos/7/arm64/20230418.1743", + "ami_name": "salt-project/ci/centos/7/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -80,9 +80,9 @@ "ssh_username": "centos" }, "centos-7": { - "ami": "ami-0dcc94e1bea829149", + "ami": "ami-0ace33028ada62ddb", "ami_description": "CI Image of CentOS 7 x86_64", - "ami_name": "salt-project/ci/centos/7/x86_64/20230418.1743", + "ami_name": "salt-project/ci/centos/7/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -90,9 +90,9 @@ "ssh_username": "centos" }, "centosstream-8-arm64": { - "ami": "ami-02783136c1080c782", + "ami": "ami-0b30827dc592b2695", "ami_description": "CI Image of CentOSStream 8 arm64", - "ami_name": "salt-project/ci/centosstream/8/arm64/20230418.1717", + "ami_name": "salt-project/ci/centosstream/8/arm64/20230522.0618", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -100,9 +100,9 @@ "ssh_username": "centos" }, "centosstream-8": { - "ami": "ami-055e35dc7180defad", + "ami": "ami-0929882a7e5cfba5f", "ami_description": "CI Image of CentOSStream 8 x86_64", - "ami_name": "salt-project/ci/centosstream/8/x86_64/20230418.1717", + "ami_name": "salt-project/ci/centosstream/8/x86_64/20230522.0618", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -110,9 +110,9 @@ "ssh_username": "centos" }, "centosstream-9-arm64": { - "ami": "ami-06fd13f7c7c702fc4", + "ami": "ami-00700fb8821b8b8c7", "ami_description": "CI Image of CentOSStream 9 arm64", - "ami_name": "salt-project/ci/centosstream/9/arm64/20230418.1717", + "ami_name": "salt-project/ci/centosstream/9/arm64/20230522.0619", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -120,9 +120,9 @@ "ssh_username": "ec2-user" }, "centosstream-9": { - "ami": "ami-044545f7a74d46acc", + "ami": "ami-0bd92f4dca5d74017", "ami_description": "CI Image of CentOSStream 9 x86_64", - "ami_name": "salt-project/ci/centosstream/9/x86_64/20230418.1717", + "ami_name": "salt-project/ci/centosstream/9/x86_64/20230522.0619", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -130,9 +130,9 @@ "ssh_username": "ec2-user" }, "debian-10-arm64": { - "ami": "ami-045aedc47e7ddfbf1", + "ami": "ami-0f681fc9d5de0c3df", "ami_description": "CI Image of Debian 10 arm64", - "ami_name": "salt-project/ci/debian/10/arm64/20230418.1739", + "ami_name": "salt-project/ci/debian/10/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -140,9 +140,9 @@ "ssh_username": "admin" }, "debian-10": { - "ami": "ami-0a205a9361210b291", + "ami": "ami-0dcf5610590139238", "ami_description": "CI Image of Debian 10 x86_64", - "ami_name": "salt-project/ci/debian/10/x86_64/20230418.1739", + "ami_name": "salt-project/ci/debian/10/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -150,9 +150,9 @@ "ssh_username": "admin" }, "debian-11-arm64": { - "ami": "ami-0be71acc27a8efa60", + "ami": "ami-062b4bf11a864825c", "ami_description": "CI Image of Debian 11 arm64", - "ami_name": "salt-project/ci/debian/11/arm64/20230418.1739", + "ami_name": "salt-project/ci/debian/11/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -160,9 +160,9 @@ "ssh_username": "admin" }, "debian-11": { - "ami": "ami-0ad354da27b34289b", + "ami": "ami-0f400e5fa6806bbca", "ami_description": "CI Image of Debian 11 x86_64", - "ami_name": "salt-project/ci/debian/11/x86_64/20230418.1742", + "ami_name": "salt-project/ci/debian/11/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -170,9 +170,9 @@ "ssh_username": "admin" }, "fedora-36-arm64": { - "ami": "ami-00c0ab2829c887922", + "ami": "ami-06dbaabd32b4c2502", "ami_description": "CI Image of Fedora 36 arm64", - "ami_name": "salt-project/ci/fedora/36/arm64/20230418.1726", + "ami_name": "salt-project/ci/fedora/36/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -180,9 +180,9 @@ "ssh_username": "fedora" }, "fedora-36": { - "ami": "ami-0185a1189bff7c771", + "ami": "ami-0b55732c36731876f", "ami_description": "CI Image of Fedora 36 x86_64", - "ami_name": "salt-project/ci/fedora/36/x86_64/20230418.1726", + "ami_name": "salt-project/ci/fedora/36/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -190,9 +190,9 @@ "ssh_username": "fedora" }, "fedora-37-arm64": { - "ami": "ami-075c52fda843ace1b", + "ami": "ami-0d71d6f2b0869842f", "ami_description": "CI Image of Fedora 37 arm64", - "ami_name": "salt-project/ci/fedora/37/arm64/20230418.1726", + "ami_name": "salt-project/ci/fedora/37/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -200,9 +200,9 @@ "ssh_username": "fedora" }, "fedora-37": { - "ami": "ami-099a68403d6c65733", + "ami": "ami-026f494dd4b9d40e8", "ami_description": "CI Image of Fedora 37 x86_64", - "ami_name": "salt-project/ci/fedora/37/x86_64/20230418.1726", + "ami_name": "salt-project/ci/fedora/37/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -210,9 +210,9 @@ "ssh_username": "fedora" }, "fedora-38-arm64": { - "ami": "ami-02fa22d081a9be052", + "ami": "ami-01ba8a7951daf68fb", "ami_description": "CI Image of Fedora 38 arm64", - "ami_name": "salt-project/ci/fedora/38/arm64/20230418.1727", + "ami_name": "salt-project/ci/fedora/38/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -220,9 +220,9 @@ "ssh_username": "fedora" }, "fedora-38": { - "ami": "ami-0a8d949d0bb15bbc0", + "ami": "ami-0699dbe70b69e96aa", "ami_description": "CI Image of Fedora 38 x86_64", - "ami_name": "salt-project/ci/fedora/38/x86_64/20230418.1727", + "ami_name": "salt-project/ci/fedora/38/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -230,9 +230,9 @@ "ssh_username": "fedora" }, "opensuse-15": { - "ami": "ami-089ac311f924f131f", + "ami": "ami-0c594da84f6e1cd96", "ami_description": "CI Image of Opensuse 15 x86_64", - "ami_name": "salt-project/ci/opensuse/15/x86_64/20230418.1744", + "ami_name": "salt-project/ci/opensuse/15/x86_64/20230522.0619", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -240,9 +240,9 @@ "ssh_username": "ec2-user" }, "photonos-3": { - "ami": "ami-03ce6db789f90957b", + "ami": "ami-0db2ebdb9bc3400ef", "ami_description": "CI Image of PhotonOS 3 x86_64", - "ami_name": "salt-project/ci/photonos/3/x86_64/20230418.1717", + "ami_name": "salt-project/ci/photonos/3/x86_64/20230522.0617", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -250,9 +250,9 @@ "ssh_username": "root" }, "photonos-4": { - "ami": "ami-0ef9996c398479d65", + "ami": "ami-08a6b6bbf6779a538", "ami_description": "CI Image of PhotonOS 4 x86_64", - "ami_name": "salt-project/ci/photonos/4/x86_64/20230418.1717", + "ami_name": "salt-project/ci/photonos/4/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -260,9 +260,9 @@ "ssh_username": "root" }, "ubuntu-20.04-arm64": { - "ami": "ami-0c4d21e0772489c0d", + "ami": "ami-0dccc0de7a38cca90", "ami_description": "CI Image of Ubuntu 20.04 arm64", - "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20230418.1728", + "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -270,9 +270,9 @@ "ssh_username": "ubuntu" }, "ubuntu-20.04": { - "ami": "ami-09ae6200865b29b9b", + "ami": "ami-05e51f893a626b579", "ami_description": "CI Image of Ubuntu 20.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20230418.1728", + "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -280,9 +280,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04-arm64": { - "ami": "ami-024fe5d0b838f88f7", + "ami": "ami-0c958272da6c09ca6", "ami_description": "CI Image of Ubuntu 22.04 arm64", - "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20230418.1731", + "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -290,9 +290,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04": { - "ami": "ami-0d83f00f084d91451", + "ami": "ami-09e45f31ccafcdcec", "ami_description": "CI Image of Ubuntu 22.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20230418.1732", + "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -300,9 +300,9 @@ "ssh_username": "ubuntu" }, "windows-2016": { - "ami": "ami-078d9229cfaf24d1b", + "ami": "ami-099db55543619f54a", "ami_description": "CI Image of Windows 2016 x86_64", - "ami_name": "salt-project/ci/windows/2016/x86_64/20230418.1717", + "ami_name": "salt-project/ci/windows/2016/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -310,9 +310,9 @@ "ssh_username": "Administrator" }, "windows-2019": { - "ami": "ami-0ab20823965e1aa7a", + "ami": "ami-0860ee5bc9ee93e13", "ami_description": "CI Image of Windows 2019 x86_64", - "ami_name": "salt-project/ci/windows/2019/x86_64/20230418.1717", + "ami_name": "salt-project/ci/windows/2019/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -320,9 +320,9 @@ "ssh_username": "Administrator" }, "windows-2022": { - "ami": "ami-054c4cf04c0f31eb1", + "ami": "ami-032e3abce2aa98da7", "ami_description": "CI Image of Windows 2022 x86_64", - "ami_name": "salt-project/ci/windows/2022/x86_64/20230418.1717", + "ami_name": "salt-project/ci/windows/2022/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", From 80ff2f662dc038f56b650e07ea0fb45a5db6bb0d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 22 May 2023 19:19:37 +0100 Subject: [PATCH 049/152] Skip `tests/unit/{modules,states}/test_zcbuildout.py` on windows. It needs special work on the golden images to get SSL to properly work. These steps are required because the code being tested is using `easy_install` which does not know how to get certificates from `certifi`. Since `easy_install` is too old, and deprecated, the extra work is not worth it, plus, they are still being tested on other platforms. Signed-off-by: Pedro Algarvio --- tests/unit/modules/test_zcbuildout.py | 9 +++++++++ tests/unit/states/test_zcbuildout.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/tests/unit/modules/test_zcbuildout.py b/tests/unit/modules/test_zcbuildout.py index f793e3fc3f8..ac98435ffa0 100644 --- a/tests/unit/modules/test_zcbuildout.py +++ b/tests/unit/modules/test_zcbuildout.py @@ -19,6 +19,15 @@ from tests.support.mixins import LoaderModuleMockMixin from tests.support.runtests import RUNTIME_VARS from tests.support.unit import TestCase +pytestmark = [ + pytest.mark.skip_on_windows( + reason=( + "Special steps are required for proper SSL validation because " + "`easy_install` is too old(and deprecated)." + ) + ) +] + KNOWN_VIRTUALENV_BINARY_NAMES = ( "virtualenv", "virtualenv2", diff --git a/tests/unit/states/test_zcbuildout.py b/tests/unit/states/test_zcbuildout.py index db6013076d1..b5f919ac6b2 100644 --- a/tests/unit/states/test_zcbuildout.py +++ b/tests/unit/states/test_zcbuildout.py @@ -10,6 +10,15 @@ import salt.utils.path from tests.support.runtests import RUNTIME_VARS from tests.unit.modules.test_zcbuildout import KNOWN_VIRTUALENV_BINARY_NAMES, Base +pytestmark = [ + pytest.mark.skip_on_windows( + reason=( + "Special steps are required for proper SSL validation because " + "`easy_install` is too old(and deprecated)." + ) + ) +] + @pytest.mark.skip_if_binaries_missing(*KNOWN_VIRTUALENV_BINARY_NAMES, check_all=False) @pytest.mark.requires_network From 6d918e15a33e9282f9dbb68760e1932dce7f26de Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 22 May 2023 19:24:59 +0100 Subject: [PATCH 050/152] Drop Fedora 36 which has reached EOL Signed-off-by: Pedro Algarvio --- .github/workflows/ci.yml | 20 ------- .github/workflows/nightly.yml | 20 ------- .github/workflows/release.yml | 40 ------------- .github/workflows/scheduled.yml | 20 ------- .github/workflows/staging.yml | 58 ------------------- .../test-pkg-repo-downloads.yml.jinja | 2 - .../templates/test-salt-pkg.yml.jinja | 1 - .../workflows/templates/test-salt.yml.jinja | 1 - cicd/golden-images.json | 20 ------- 9 files changed, 182 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a286234889a..11214d88d51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1138,25 +1138,6 @@ jobs: skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} - fedora-36: - name: Fedora 36 - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-salt-onedir - uses: ./.github/workflows/test-action.yml - with: - distro-slug: fedora-36 - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} - skip-code-coverage: ${{ github.event_name == 'pull_request' }} - skip-junit-reports: ${{ github.event_name == 'pull_request' }} - fedora-37: name: Fedora 37 if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -1355,7 +1336,6 @@ jobs: - debian-10 - debian-11 - debian-11-arm64 - - fedora-36 - fedora-37 - fedora-38 - opensuse-15 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1bdcfe65086..cf41d2e358e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1197,25 +1197,6 @@ jobs: skip-code-coverage: false skip-junit-reports: false - fedora-36: - name: Fedora 36 - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-salt-onedir - uses: ./.github/workflows/test-action.yml - with: - distro-slug: fedora-36 - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} - skip-code-coverage: false - skip-junit-reports: false - fedora-37: name: Fedora 37 if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -2046,7 +2027,6 @@ jobs: - debian-10 - debian-11 - debian-11-arm64 - - fedora-36 - fedora-37 - fedora-38 - opensuse-15 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 61fc1f5783e..f121d380e0a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -505,44 +505,6 @@ jobs: latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}" secrets: inherit - fedora-36-pkg-download-tests: - name: Test Fedora 36 Package Downloads - if: ${{ inputs.skip-salt-pkg-download-test-suite == false }} - needs: - - prepare-workflow - - publish-repositories - - download-onedir-artifact - uses: ./.github/workflows/test-package-downloads-action-linux.yml - with: - distro-slug: fedora-36 - platform: linux - arch: x86_64 - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - environment: release - skip-code-coverage: true - latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}" - secrets: inherit - - fedora-36-arm64-pkg-download-tests: - name: Test Fedora 36 Arm64 Package Downloads - if: ${{ inputs.skip-salt-pkg-download-test-suite == false }} - needs: - - prepare-workflow - - publish-repositories - - download-onedir-artifact - uses: ./.github/workflows/test-package-downloads-action-linux.yml - with: - distro-slug: fedora-36-arm64 - platform: linux - arch: aarch64 - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - environment: release - skip-code-coverage: true - latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}" - secrets: inherit - fedora-37-pkg-download-tests: name: Test Fedora 37 Package Downloads if: ${{ inputs.skip-salt-pkg-download-test-suite == false }} @@ -818,8 +780,6 @@ jobs: - debian-10-pkg-download-tests - debian-11-pkg-download-tests - debian-11-arm64-pkg-download-tests - - fedora-36-pkg-download-tests - - fedora-36-arm64-pkg-download-tests - fedora-37-pkg-download-tests - fedora-37-arm64-pkg-download-tests - fedora-38-pkg-download-tests diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 368089af2df..8e725fcd9b2 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -1181,25 +1181,6 @@ jobs: skip-code-coverage: false skip-junit-reports: false - fedora-36: - name: Fedora 36 - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-salt-onedir - uses: ./.github/workflows/test-action.yml - with: - distro-slug: fedora-36 - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} - skip-code-coverage: false - skip-junit-reports: false - fedora-37: name: Fedora 37 if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -1400,7 +1381,6 @@ jobs: - debian-10 - debian-11 - debian-11-arm64 - - fedora-36 - fedora-37 - fedora-38 - opensuse-15 diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 424f47c363e..7e7a492dfd4 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -1187,25 +1187,6 @@ jobs: skip-code-coverage: true skip-junit-reports: true - fedora-36: - name: Fedora 36 - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-salt-onedir - uses: ./.github/workflows/test-action.yml - with: - distro-slug: fedora-36 - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} - skip-code-coverage: true - skip-junit-reports: true - fedora-37: name: Fedora 37 if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -2394,42 +2375,6 @@ jobs: latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}" secrets: inherit - fedora-36-pkg-download-tests: - name: Test Fedora 36 Package Downloads - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test-pkg-download'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - publish-repositories - uses: ./.github/workflows/test-package-downloads-action-linux.yml - with: - distro-slug: fedora-36 - platform: linux - arch: x86_64 - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - environment: staging - skip-code-coverage: true - latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}" - secrets: inherit - - fedora-36-arm64-pkg-download-tests: - name: Test Fedora 36 Arm64 Package Downloads - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test-pkg-download'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - publish-repositories - uses: ./.github/workflows/test-package-downloads-action-linux.yml - with: - distro-slug: fedora-36-arm64 - platform: linux - arch: aarch64 - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - environment: staging - skip-code-coverage: true - latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}" - secrets: inherit - fedora-37-pkg-download-tests: name: Test Fedora 37 Package Downloads if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test-pkg-download'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -2686,7 +2631,6 @@ jobs: - debian-10 - debian-11 - debian-11-arm64 - - fedora-36 - fedora-37 - fedora-38 - opensuse-15 @@ -2731,8 +2675,6 @@ jobs: - debian-10-pkg-download-tests - debian-11-pkg-download-tests - debian-11-arm64-pkg-download-tests - - fedora-36-pkg-download-tests - - fedora-36-arm64-pkg-download-tests - fedora-37-pkg-download-tests - fedora-37-arm64-pkg-download-tests - fedora-38-pkg-download-tests diff --git a/.github/workflows/templates/test-pkg-repo-downloads.yml.jinja b/.github/workflows/templates/test-pkg-repo-downloads.yml.jinja index ac826f6e9fe..8ea9bfed3b7 100644 --- a/.github/workflows/templates/test-pkg-repo-downloads.yml.jinja +++ b/.github/workflows/templates/test-pkg-repo-downloads.yml.jinja @@ -15,8 +15,6 @@ ("debian-10", "Debian 10", "x86_64"), ("debian-11", "Debian 11", "x86_64"), ("debian-11-arm64", "Debian 11 Arm64", "aarch64"), - ("fedora-36", "Fedora 36", "x86_64"), - ("fedora-36-arm64", "Fedora 36 Arm64", "aarch64"), ("fedora-37", "Fedora 37", "x86_64"), ("fedora-37-arm64", "Fedora 37 Arm64", "aarch64"), ("fedora-38", "Fedora 38", "x86_64"), diff --git a/.github/workflows/templates/test-salt-pkg.yml.jinja b/.github/workflows/templates/test-salt-pkg.yml.jinja index 99fc85db4fb..3970ac3d167 100644 --- a/.github/workflows/templates/test-salt-pkg.yml.jinja +++ b/.github/workflows/templates/test-salt-pkg.yml.jinja @@ -9,7 +9,6 @@ ("debian-10", "Debian 10", "x86_64", "deb"), ("debian-11", "Debian 11", "x86_64", "deb"), ("debian-11-arm64", "Debian 11 Arm64", "aarch64", "deb"), - ("fedora-36", "Fedora 36", "x86_64", "rpm"), ("fedora-37", "Fedora 37", "x86_64", "rpm"), ("fedora-38", "Fedora 38", "x86_64", "rpm"), ("ubuntu-20.04", "Ubuntu 20.04", "x86_64", "deb"), diff --git a/.github/workflows/templates/test-salt.yml.jinja b/.github/workflows/templates/test-salt.yml.jinja index 2eb0fb5e50e..6ae89e0cb16 100644 --- a/.github/workflows/templates/test-salt.yml.jinja +++ b/.github/workflows/templates/test-salt.yml.jinja @@ -59,7 +59,6 @@ ("debian-10", "Debian 10", "x86_64"), ("debian-11", "Debian 11", "x86_64"), ("debian-11-arm64", "Debian 11 Arm64", "aarch64"), - ("fedora-36", "Fedora 36", "x86_64"), ("fedora-37", "Fedora 37", "x86_64"), ("fedora-38", "Fedora 38", "x86_64"), ("opensuse-15", "Opensuse 15", "x86_64"), diff --git a/cicd/golden-images.json b/cicd/golden-images.json index 02c3ee0977c..21c702ca732 100644 --- a/cicd/golden-images.json +++ b/cicd/golden-images.json @@ -169,26 +169,6 @@ "is_windows": "false", "ssh_username": "admin" }, - "fedora-36-arm64": { - "ami": "ami-06dbaabd32b4c2502", - "ami_description": "CI Image of Fedora 36 arm64", - "ami_name": "salt-project/ci/fedora/36/arm64/20230522.0606", - "arch": "arm64", - "cloudwatch-agent-available": "true", - "instance_type": "m6g.large", - "is_windows": "false", - "ssh_username": "fedora" - }, - "fedora-36": { - "ami": "ami-0b55732c36731876f", - "ami_description": "CI Image of Fedora 36 x86_64", - "ami_name": "salt-project/ci/fedora/36/x86_64/20230522.0606", - "arch": "x86_64", - "cloudwatch-agent-available": "true", - "instance_type": "t3a.large", - "is_windows": "false", - "ssh_username": "fedora" - }, "fedora-37-arm64": { "ami": "ami-0d71d6f2b0869842f", "ami_description": "CI Image of Fedora 37 arm64", From b426bd1216d497eea3ff9964ab74e96e4bddafca Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 22 May 2023 11:05:12 +0100 Subject: [PATCH 051/152] Update to latest golden images Signed-off-by: Pedro Algarvio --- cicd/amis.yml | 2 +- cicd/golden-images.json | 132 ++++++++++++++++++++-------------------- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/cicd/amis.yml b/cicd/amis.yml index 47edcf0184b..8fb4513180f 100644 --- a/cicd/amis.yml +++ b/cicd/amis.yml @@ -1 +1 @@ -centosstream-9-x86_64: ami-044545f7a74d46acc +centosstream-9-x86_64: ami-0bd92f4dca5d74017 diff --git a/cicd/golden-images.json b/cicd/golden-images.json index 75341e64aeb..02c3ee0977c 100644 --- a/cicd/golden-images.json +++ b/cicd/golden-images.json @@ -1,8 +1,8 @@ { "almalinux-8-arm64": { - "ami": "ami-0fc1e14bf9ff422aa", + "ami": "ami-05c1d3dbdeeb94bc6", "ami_description": "CI Image of AlmaLinux 8 arm64", - "ami_name": "salt-project/ci/almalinux/8/arm64/20230418.1731", + "ami_name": "salt-project/ci/almalinux/8/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -10,9 +10,9 @@ "ssh_username": "ec2-user" }, "almalinux-8": { - "ami": "ami-0bae4158c1f126271", + "ami": "ami-0ec1cbc531f10105b", "ami_description": "CI Image of AlmaLinux 8 x86_64", - "ami_name": "salt-project/ci/almalinux/8/x86_64/20230418.1732", + "ami_name": "salt-project/ci/almalinux/8/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -20,9 +20,9 @@ "ssh_username": "ec2-user" }, "almalinux-9-arm64": { - "ami": "ami-08f4d0fbf5d53c3ab", + "ami": "ami-036c495af9dfcf852", "ami_description": "CI Image of AlmaLinux 9 arm64", - "ami_name": "salt-project/ci/almalinux/9/arm64/20230418.1732", + "ami_name": "salt-project/ci/almalinux/9/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -30,9 +30,9 @@ "ssh_username": "ec2-user" }, "almalinux-9": { - "ami": "ami-00404c1cc5c5a08bd", + "ami": "ami-0dbc7030666419671", "ami_description": "CI Image of AlmaLinux 9 x86_64", - "ami_name": "salt-project/ci/almalinux/9/x86_64/20230418.1738", + "ami_name": "salt-project/ci/almalinux/9/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -40,9 +40,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2-arm64": { - "ami": "ami-05fbdb644d06c27b6", + "ami": "ami-022232915c2a5f2d0", "ami_description": "CI Image of AmazonLinux 2 arm64", - "ami_name": "salt-project/ci/amazonlinux/2/arm64/20230418.1717", + "ami_name": "salt-project/ci/amazonlinux/2/arm64/20230522.0621", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -50,9 +50,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2": { - "ami": "ami-014171e6c30ec8387", + "ami": "ami-0695f87baa5b5ce15", "ami_description": "CI Image of AmazonLinux 2 x86_64", - "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20230418.1718", + "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20230522.0620", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -60,9 +60,9 @@ "ssh_username": "ec2-user" }, "archlinux-lts": { - "ami": "ami-00cff81ed2e2fb0f4", + "ami": "ami-0f6424847f98afc04", "ami_description": "CI Image of ArchLinux lts x86_64", - "ami_name": "salt-project/ci/archlinux/lts/x86_64/20230418.1717", + "ami_name": "salt-project/ci/archlinux/lts/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "false", "instance_type": "t3a.large", @@ -70,9 +70,9 @@ "ssh_username": "arch" }, "centos-7-arm64": { - "ami": "ami-051cef43c13fcc0c9", + "ami": "ami-0908831c364e33a37", "ami_description": "CI Image of CentOS 7 arm64", - "ami_name": "salt-project/ci/centos/7/arm64/20230418.1743", + "ami_name": "salt-project/ci/centos/7/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -80,9 +80,9 @@ "ssh_username": "centos" }, "centos-7": { - "ami": "ami-0dcc94e1bea829149", + "ami": "ami-0ace33028ada62ddb", "ami_description": "CI Image of CentOS 7 x86_64", - "ami_name": "salt-project/ci/centos/7/x86_64/20230418.1743", + "ami_name": "salt-project/ci/centos/7/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -90,9 +90,9 @@ "ssh_username": "centos" }, "centosstream-8-arm64": { - "ami": "ami-02783136c1080c782", + "ami": "ami-0b30827dc592b2695", "ami_description": "CI Image of CentOSStream 8 arm64", - "ami_name": "salt-project/ci/centosstream/8/arm64/20230418.1717", + "ami_name": "salt-project/ci/centosstream/8/arm64/20230522.0618", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -100,9 +100,9 @@ "ssh_username": "centos" }, "centosstream-8": { - "ami": "ami-055e35dc7180defad", + "ami": "ami-0929882a7e5cfba5f", "ami_description": "CI Image of CentOSStream 8 x86_64", - "ami_name": "salt-project/ci/centosstream/8/x86_64/20230418.1717", + "ami_name": "salt-project/ci/centosstream/8/x86_64/20230522.0618", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -110,9 +110,9 @@ "ssh_username": "centos" }, "centosstream-9-arm64": { - "ami": "ami-06fd13f7c7c702fc4", + "ami": "ami-00700fb8821b8b8c7", "ami_description": "CI Image of CentOSStream 9 arm64", - "ami_name": "salt-project/ci/centosstream/9/arm64/20230418.1717", + "ami_name": "salt-project/ci/centosstream/9/arm64/20230522.0619", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -120,9 +120,9 @@ "ssh_username": "ec2-user" }, "centosstream-9": { - "ami": "ami-044545f7a74d46acc", + "ami": "ami-0bd92f4dca5d74017", "ami_description": "CI Image of CentOSStream 9 x86_64", - "ami_name": "salt-project/ci/centosstream/9/x86_64/20230418.1717", + "ami_name": "salt-project/ci/centosstream/9/x86_64/20230522.0619", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -130,9 +130,9 @@ "ssh_username": "ec2-user" }, "debian-10-arm64": { - "ami": "ami-045aedc47e7ddfbf1", + "ami": "ami-0f681fc9d5de0c3df", "ami_description": "CI Image of Debian 10 arm64", - "ami_name": "salt-project/ci/debian/10/arm64/20230418.1739", + "ami_name": "salt-project/ci/debian/10/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -140,9 +140,9 @@ "ssh_username": "admin" }, "debian-10": { - "ami": "ami-0a205a9361210b291", + "ami": "ami-0dcf5610590139238", "ami_description": "CI Image of Debian 10 x86_64", - "ami_name": "salt-project/ci/debian/10/x86_64/20230418.1739", + "ami_name": "salt-project/ci/debian/10/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -150,9 +150,9 @@ "ssh_username": "admin" }, "debian-11-arm64": { - "ami": "ami-0be71acc27a8efa60", + "ami": "ami-062b4bf11a864825c", "ami_description": "CI Image of Debian 11 arm64", - "ami_name": "salt-project/ci/debian/11/arm64/20230418.1739", + "ami_name": "salt-project/ci/debian/11/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -160,9 +160,9 @@ "ssh_username": "admin" }, "debian-11": { - "ami": "ami-0ad354da27b34289b", + "ami": "ami-0f400e5fa6806bbca", "ami_description": "CI Image of Debian 11 x86_64", - "ami_name": "salt-project/ci/debian/11/x86_64/20230418.1742", + "ami_name": "salt-project/ci/debian/11/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -170,9 +170,9 @@ "ssh_username": "admin" }, "fedora-36-arm64": { - "ami": "ami-00c0ab2829c887922", + "ami": "ami-06dbaabd32b4c2502", "ami_description": "CI Image of Fedora 36 arm64", - "ami_name": "salt-project/ci/fedora/36/arm64/20230418.1726", + "ami_name": "salt-project/ci/fedora/36/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -180,9 +180,9 @@ "ssh_username": "fedora" }, "fedora-36": { - "ami": "ami-0185a1189bff7c771", + "ami": "ami-0b55732c36731876f", "ami_description": "CI Image of Fedora 36 x86_64", - "ami_name": "salt-project/ci/fedora/36/x86_64/20230418.1726", + "ami_name": "salt-project/ci/fedora/36/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -190,9 +190,9 @@ "ssh_username": "fedora" }, "fedora-37-arm64": { - "ami": "ami-075c52fda843ace1b", + "ami": "ami-0d71d6f2b0869842f", "ami_description": "CI Image of Fedora 37 arm64", - "ami_name": "salt-project/ci/fedora/37/arm64/20230418.1726", + "ami_name": "salt-project/ci/fedora/37/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -200,9 +200,9 @@ "ssh_username": "fedora" }, "fedora-37": { - "ami": "ami-099a68403d6c65733", + "ami": "ami-026f494dd4b9d40e8", "ami_description": "CI Image of Fedora 37 x86_64", - "ami_name": "salt-project/ci/fedora/37/x86_64/20230418.1726", + "ami_name": "salt-project/ci/fedora/37/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -210,9 +210,9 @@ "ssh_username": "fedora" }, "fedora-38-arm64": { - "ami": "ami-02fa22d081a9be052", + "ami": "ami-01ba8a7951daf68fb", "ami_description": "CI Image of Fedora 38 arm64", - "ami_name": "salt-project/ci/fedora/38/arm64/20230418.1727", + "ami_name": "salt-project/ci/fedora/38/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -220,9 +220,9 @@ "ssh_username": "fedora" }, "fedora-38": { - "ami": "ami-0a8d949d0bb15bbc0", + "ami": "ami-0699dbe70b69e96aa", "ami_description": "CI Image of Fedora 38 x86_64", - "ami_name": "salt-project/ci/fedora/38/x86_64/20230418.1727", + "ami_name": "salt-project/ci/fedora/38/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -230,9 +230,9 @@ "ssh_username": "fedora" }, "opensuse-15": { - "ami": "ami-089ac311f924f131f", + "ami": "ami-0c594da84f6e1cd96", "ami_description": "CI Image of Opensuse 15 x86_64", - "ami_name": "salt-project/ci/opensuse/15/x86_64/20230418.1744", + "ami_name": "salt-project/ci/opensuse/15/x86_64/20230522.0619", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -240,9 +240,9 @@ "ssh_username": "ec2-user" }, "photonos-3": { - "ami": "ami-03ce6db789f90957b", + "ami": "ami-0db2ebdb9bc3400ef", "ami_description": "CI Image of PhotonOS 3 x86_64", - "ami_name": "salt-project/ci/photonos/3/x86_64/20230418.1717", + "ami_name": "salt-project/ci/photonos/3/x86_64/20230522.0617", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -250,9 +250,9 @@ "ssh_username": "root" }, "photonos-4": { - "ami": "ami-0ef9996c398479d65", + "ami": "ami-08a6b6bbf6779a538", "ami_description": "CI Image of PhotonOS 4 x86_64", - "ami_name": "salt-project/ci/photonos/4/x86_64/20230418.1717", + "ami_name": "salt-project/ci/photonos/4/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -260,9 +260,9 @@ "ssh_username": "root" }, "ubuntu-20.04-arm64": { - "ami": "ami-0c4d21e0772489c0d", + "ami": "ami-0dccc0de7a38cca90", "ami_description": "CI Image of Ubuntu 20.04 arm64", - "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20230418.1728", + "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -270,9 +270,9 @@ "ssh_username": "ubuntu" }, "ubuntu-20.04": { - "ami": "ami-09ae6200865b29b9b", + "ami": "ami-05e51f893a626b579", "ami_description": "CI Image of Ubuntu 20.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20230418.1728", + "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -280,9 +280,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04-arm64": { - "ami": "ami-024fe5d0b838f88f7", + "ami": "ami-0c958272da6c09ca6", "ami_description": "CI Image of Ubuntu 22.04 arm64", - "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20230418.1731", + "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20230522.0606", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -290,9 +290,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04": { - "ami": "ami-0d83f00f084d91451", + "ami": "ami-09e45f31ccafcdcec", "ami_description": "CI Image of Ubuntu 22.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20230418.1732", + "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -300,9 +300,9 @@ "ssh_username": "ubuntu" }, "windows-2016": { - "ami": "ami-078d9229cfaf24d1b", + "ami": "ami-099db55543619f54a", "ami_description": "CI Image of Windows 2016 x86_64", - "ami_name": "salt-project/ci/windows/2016/x86_64/20230418.1717", + "ami_name": "salt-project/ci/windows/2016/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -310,9 +310,9 @@ "ssh_username": "Administrator" }, "windows-2019": { - "ami": "ami-0ab20823965e1aa7a", + "ami": "ami-0860ee5bc9ee93e13", "ami_description": "CI Image of Windows 2019 x86_64", - "ami_name": "salt-project/ci/windows/2019/x86_64/20230418.1717", + "ami_name": "salt-project/ci/windows/2019/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -320,9 +320,9 @@ "ssh_username": "Administrator" }, "windows-2022": { - "ami": "ami-054c4cf04c0f31eb1", + "ami": "ami-032e3abce2aa98da7", "ami_description": "CI Image of Windows 2022 x86_64", - "ami_name": "salt-project/ci/windows/2022/x86_64/20230418.1717", + "ami_name": "salt-project/ci/windows/2022/x86_64/20230522.0606", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", From 19025ad56838ba80d1b61e6fcb5e2672b94faa31 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 22 May 2023 19:19:37 +0100 Subject: [PATCH 052/152] Skip `tests/unit/{modules,states}/test_zcbuildout.py` on windows. It needs special work on the golden images to get SSL to properly work. These steps are required because the code being tested is using `easy_install` which does not know how to get certificates from `certifi`. Since `easy_install` is too old, and deprecated, the extra work is not worth it, plus, they are still being tested on other platforms. Signed-off-by: Pedro Algarvio --- tests/unit/modules/test_zcbuildout.py | 9 +++++++++ tests/unit/states/test_zcbuildout.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/tests/unit/modules/test_zcbuildout.py b/tests/unit/modules/test_zcbuildout.py index f793e3fc3f8..ac98435ffa0 100644 --- a/tests/unit/modules/test_zcbuildout.py +++ b/tests/unit/modules/test_zcbuildout.py @@ -19,6 +19,15 @@ from tests.support.mixins import LoaderModuleMockMixin from tests.support.runtests import RUNTIME_VARS from tests.support.unit import TestCase +pytestmark = [ + pytest.mark.skip_on_windows( + reason=( + "Special steps are required for proper SSL validation because " + "`easy_install` is too old(and deprecated)." + ) + ) +] + KNOWN_VIRTUALENV_BINARY_NAMES = ( "virtualenv", "virtualenv2", diff --git a/tests/unit/states/test_zcbuildout.py b/tests/unit/states/test_zcbuildout.py index db6013076d1..b5f919ac6b2 100644 --- a/tests/unit/states/test_zcbuildout.py +++ b/tests/unit/states/test_zcbuildout.py @@ -10,6 +10,15 @@ import salt.utils.path from tests.support.runtests import RUNTIME_VARS from tests.unit.modules.test_zcbuildout import KNOWN_VIRTUALENV_BINARY_NAMES, Base +pytestmark = [ + pytest.mark.skip_on_windows( + reason=( + "Special steps are required for proper SSL validation because " + "`easy_install` is too old(and deprecated)." + ) + ) +] + @pytest.mark.skip_if_binaries_missing(*KNOWN_VIRTUALENV_BINARY_NAMES, check_all=False) @pytest.mark.requires_network From bc93320135a061fa023ccb085f243b51e72f93e3 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 22 May 2023 19:24:59 +0100 Subject: [PATCH 053/152] Drop Fedora 36 which has reached EOL Signed-off-by: Pedro Algarvio --- .github/workflows/ci.yml | 20 ------- .github/workflows/nightly.yml | 20 ------- .github/workflows/release.yml | 40 ------------- .github/workflows/scheduled.yml | 20 ------- .github/workflows/staging.yml | 58 ------------------- .../test-pkg-repo-downloads.yml.jinja | 2 - .../templates/test-salt-pkg.yml.jinja | 1 - .../workflows/templates/test-salt.yml.jinja | 1 - cicd/golden-images.json | 20 ------- 9 files changed, 182 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a18e21fcc5d..310925bc823 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1117,25 +1117,6 @@ jobs: skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} - fedora-36: - name: Fedora 36 - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-salt-onedir - uses: ./.github/workflows/test-action.yml - with: - distro-slug: fedora-36 - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} - skip-code-coverage: ${{ github.event_name == 'pull_request' }} - skip-junit-reports: ${{ github.event_name == 'pull_request' }} - fedora-37: name: Fedora 37 if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -1334,7 +1315,6 @@ jobs: - debian-10 - debian-11 - debian-11-arm64 - - fedora-36 - fedora-37 - fedora-38 - opensuse-15 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8291efe30fa..3e91af10b1a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1181,25 +1181,6 @@ jobs: skip-code-coverage: false skip-junit-reports: false - fedora-36: - name: Fedora 36 - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-salt-onedir - uses: ./.github/workflows/test-action.yml - with: - distro-slug: fedora-36 - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} - skip-code-coverage: false - skip-junit-reports: false - fedora-37: name: Fedora 37 if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -2030,7 +2011,6 @@ jobs: - debian-10 - debian-11 - debian-11-arm64 - - fedora-36 - fedora-37 - fedora-38 - opensuse-15 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 61fc1f5783e..f121d380e0a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -505,44 +505,6 @@ jobs: latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}" secrets: inherit - fedora-36-pkg-download-tests: - name: Test Fedora 36 Package Downloads - if: ${{ inputs.skip-salt-pkg-download-test-suite == false }} - needs: - - prepare-workflow - - publish-repositories - - download-onedir-artifact - uses: ./.github/workflows/test-package-downloads-action-linux.yml - with: - distro-slug: fedora-36 - platform: linux - arch: x86_64 - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - environment: release - skip-code-coverage: true - latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}" - secrets: inherit - - fedora-36-arm64-pkg-download-tests: - name: Test Fedora 36 Arm64 Package Downloads - if: ${{ inputs.skip-salt-pkg-download-test-suite == false }} - needs: - - prepare-workflow - - publish-repositories - - download-onedir-artifact - uses: ./.github/workflows/test-package-downloads-action-linux.yml - with: - distro-slug: fedora-36-arm64 - platform: linux - arch: aarch64 - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - environment: release - skip-code-coverage: true - latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}" - secrets: inherit - fedora-37-pkg-download-tests: name: Test Fedora 37 Package Downloads if: ${{ inputs.skip-salt-pkg-download-test-suite == false }} @@ -818,8 +780,6 @@ jobs: - debian-10-pkg-download-tests - debian-11-pkg-download-tests - debian-11-arm64-pkg-download-tests - - fedora-36-pkg-download-tests - - fedora-36-arm64-pkg-download-tests - fedora-37-pkg-download-tests - fedora-37-arm64-pkg-download-tests - fedora-38-pkg-download-tests diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index fda566fbb3e..8a8d9af83ae 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -1160,25 +1160,6 @@ jobs: skip-code-coverage: false skip-junit-reports: false - fedora-36: - name: Fedora 36 - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-salt-onedir - uses: ./.github/workflows/test-action.yml - with: - distro-slug: fedora-36 - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} - skip-code-coverage: false - skip-junit-reports: false - fedora-37: name: Fedora 37 if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -1379,7 +1360,6 @@ jobs: - debian-10 - debian-11 - debian-11-arm64 - - fedora-36 - fedora-37 - fedora-38 - opensuse-15 diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 2088976ec31..450f0f5b64b 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -1166,25 +1166,6 @@ jobs: skip-code-coverage: true skip-junit-reports: true - fedora-36: - name: Fedora 36 - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-salt-onedir - uses: ./.github/workflows/test-action.yml - with: - distro-slug: fedora-36 - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} - skip-code-coverage: true - skip-junit-reports: true - fedora-37: name: Fedora 37 if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -2373,42 +2354,6 @@ jobs: latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}" secrets: inherit - fedora-36-pkg-download-tests: - name: Test Fedora 36 Package Downloads - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test-pkg-download'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - publish-repositories - uses: ./.github/workflows/test-package-downloads-action-linux.yml - with: - distro-slug: fedora-36 - platform: linux - arch: x86_64 - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - environment: staging - skip-code-coverage: true - latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}" - secrets: inherit - - fedora-36-arm64-pkg-download-tests: - name: Test Fedora 36 Arm64 Package Downloads - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test-pkg-download'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - publish-repositories - uses: ./.github/workflows/test-package-downloads-action-linux.yml - with: - distro-slug: fedora-36-arm64 - platform: linux - arch: aarch64 - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - environment: staging - skip-code-coverage: true - latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}" - secrets: inherit - fedora-37-pkg-download-tests: name: Test Fedora 37 Package Downloads if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test-pkg-download'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -2665,7 +2610,6 @@ jobs: - debian-10 - debian-11 - debian-11-arm64 - - fedora-36 - fedora-37 - fedora-38 - opensuse-15 @@ -2710,8 +2654,6 @@ jobs: - debian-10-pkg-download-tests - debian-11-pkg-download-tests - debian-11-arm64-pkg-download-tests - - fedora-36-pkg-download-tests - - fedora-36-arm64-pkg-download-tests - fedora-37-pkg-download-tests - fedora-37-arm64-pkg-download-tests - fedora-38-pkg-download-tests diff --git a/.github/workflows/templates/test-pkg-repo-downloads.yml.jinja b/.github/workflows/templates/test-pkg-repo-downloads.yml.jinja index ac826f6e9fe..8ea9bfed3b7 100644 --- a/.github/workflows/templates/test-pkg-repo-downloads.yml.jinja +++ b/.github/workflows/templates/test-pkg-repo-downloads.yml.jinja @@ -15,8 +15,6 @@ ("debian-10", "Debian 10", "x86_64"), ("debian-11", "Debian 11", "x86_64"), ("debian-11-arm64", "Debian 11 Arm64", "aarch64"), - ("fedora-36", "Fedora 36", "x86_64"), - ("fedora-36-arm64", "Fedora 36 Arm64", "aarch64"), ("fedora-37", "Fedora 37", "x86_64"), ("fedora-37-arm64", "Fedora 37 Arm64", "aarch64"), ("fedora-38", "Fedora 38", "x86_64"), diff --git a/.github/workflows/templates/test-salt-pkg.yml.jinja b/.github/workflows/templates/test-salt-pkg.yml.jinja index 99fc85db4fb..3970ac3d167 100644 --- a/.github/workflows/templates/test-salt-pkg.yml.jinja +++ b/.github/workflows/templates/test-salt-pkg.yml.jinja @@ -9,7 +9,6 @@ ("debian-10", "Debian 10", "x86_64", "deb"), ("debian-11", "Debian 11", "x86_64", "deb"), ("debian-11-arm64", "Debian 11 Arm64", "aarch64", "deb"), - ("fedora-36", "Fedora 36", "x86_64", "rpm"), ("fedora-37", "Fedora 37", "x86_64", "rpm"), ("fedora-38", "Fedora 38", "x86_64", "rpm"), ("ubuntu-20.04", "Ubuntu 20.04", "x86_64", "deb"), diff --git a/.github/workflows/templates/test-salt.yml.jinja b/.github/workflows/templates/test-salt.yml.jinja index 2eb0fb5e50e..6ae89e0cb16 100644 --- a/.github/workflows/templates/test-salt.yml.jinja +++ b/.github/workflows/templates/test-salt.yml.jinja @@ -59,7 +59,6 @@ ("debian-10", "Debian 10", "x86_64"), ("debian-11", "Debian 11", "x86_64"), ("debian-11-arm64", "Debian 11 Arm64", "aarch64"), - ("fedora-36", "Fedora 36", "x86_64"), ("fedora-37", "Fedora 37", "x86_64"), ("fedora-38", "Fedora 38", "x86_64"), ("opensuse-15", "Opensuse 15", "x86_64"), diff --git a/cicd/golden-images.json b/cicd/golden-images.json index 02c3ee0977c..21c702ca732 100644 --- a/cicd/golden-images.json +++ b/cicd/golden-images.json @@ -169,26 +169,6 @@ "is_windows": "false", "ssh_username": "admin" }, - "fedora-36-arm64": { - "ami": "ami-06dbaabd32b4c2502", - "ami_description": "CI Image of Fedora 36 arm64", - "ami_name": "salt-project/ci/fedora/36/arm64/20230522.0606", - "arch": "arm64", - "cloudwatch-agent-available": "true", - "instance_type": "m6g.large", - "is_windows": "false", - "ssh_username": "fedora" - }, - "fedora-36": { - "ami": "ami-0b55732c36731876f", - "ami_description": "CI Image of Fedora 36 x86_64", - "ami_name": "salt-project/ci/fedora/36/x86_64/20230522.0606", - "arch": "x86_64", - "cloudwatch-agent-available": "true", - "instance_type": "t3a.large", - "is_windows": "false", - "ssh_username": "fedora" - }, "fedora-37-arm64": { "ami": "ami-0d71d6f2b0869842f", "ami_description": "CI Image of Fedora 37 arm64", From dc52847a399365814747e84142a72bebc5472b76 Mon Sep 17 00:00:00 2001 From: Megan Wilhite Date: Mon, 22 May 2023 08:33:03 -0600 Subject: [PATCH 054/152] Raise error when cannot add key with aptpkg --- salt/modules/aptpkg.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index f68b1907e88..2ec69c58898 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -632,7 +632,7 @@ def install( reinstall=False, downloadonly=False, ignore_epoch=False, - **kwargs + **kwargs, ): """ .. versionchanged:: 2015.8.12,2016.3.3,2016.11.0 @@ -2813,13 +2813,17 @@ def mod_repo(repo, saltenv="base", aptkey=True, **kwargs): else: if not aptkey: key_file = kwargs["signedby"] - add_repo_key( + if not add_repo_key( keyid=key, keyserver=keyserver, aptkey=False, keydir=key_file.parent, keyfile=key_file, - ) + ): + raise CommandExecutionError( + f"Error: Could not add key: {key}" + ) + else: cmd = [ "apt-key", @@ -2859,7 +2863,7 @@ def mod_repo(repo, saltenv="base", aptkey=True, **kwargs): func_kwargs["keydir"] = kwargs.get("signedby").parent if not add_repo_key(path=str(fn_), aptkey=False, **func_kwargs): - return False + raise CommandExecutionError(f"Error: Could not add key: {str(fn_)}") else: cmd = ["apt-key", "add", str(fn_)] out = __salt__["cmd.run_stdout"](cmd, python_shell=False, **kwargs) From 00c97d6865c1a737907d2fb9fbaa148019b829b0 Mon Sep 17 00:00:00 2001 From: Megan Wilhite Date: Mon, 22 May 2023 12:57:40 -0600 Subject: [PATCH 055/152] Add test coverage and changelog --- changelog/64253.fixed.md | 1 + .../functional/states/pkgrepo/test_debian.py | 55 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 changelog/64253.fixed.md diff --git a/changelog/64253.fixed.md b/changelog/64253.fixed.md new file mode 100644 index 00000000000..b121f3917b6 --- /dev/null +++ b/changelog/64253.fixed.md @@ -0,0 +1 @@ +Ensure we return an error when adding the key fails in the pkgrepo state for debian hosts. diff --git a/tests/pytests/functional/states/pkgrepo/test_debian.py b/tests/pytests/functional/states/pkgrepo/test_debian.py index d025643aa4c..c6ef04ba286 100644 --- a/tests/pytests/functional/states/pkgrepo/test_debian.py +++ b/tests/pytests/functional/states/pkgrepo/test_debian.py @@ -10,8 +10,10 @@ import _pytest._version import attr import pytest +import salt.modules.aptpkg import salt.utils.files from tests.conftest import CODE_DIR +from tests.support.mock import MagicMock, patch PYTEST_GE_7 = getattr(_pytest._version, "version_tuple", (-1, -1)) >= (7, 0) @@ -789,3 +791,56 @@ def test_adding_repo_file_signedby_alt_file(pkgrepo, states, repo): assert file_content.endswith("\n") assert key_file.is_file() assert repo_content in ret.comment + + +def test_adding_repo_file_signedby_fail_key_keyid( + pkgrepo, states, repo, subtests, modules +): + """ + Test adding a repo file using pkgrepo.managed + and setting signedby and keyid when adding the key fails + an error is returned + """ + + def _run(test=False): + return states.pkgrepo.managed( + name=repo.repo_content, + file=str(repo.repo_file), + clean_file=True, + signedby=str(repo.key_file), + keyid="10857FFDD3F91EAE577A21D664CBBC8173D76B3F1", + keyserver="keyserver.ubuntu.com", + aptkey=False, + test=test, + keydir="/tmp/test", + ) + + ret = _run() + assert "Failed to configure repo" in ret.comment + assert "Could not add key" in ret.comment + + +def test_adding_repo_file_signedby_fail_key_keyurl( + pkgrepo, states, repo, subtests, modules +): + """ + Test adding a repo file using pkgrepo.managed + and setting signedby and keyurl when adding the key fails + an error is returned + """ + + def _run(test=False): + with patch( + "salt.utils.path.which", MagicMock(side_effect=[True, True, False, False]) + ): + return states.pkgrepo.managed( + name=repo.repo_content, + file=str(repo.repo_file), + clean_file=True, + key_url="https://repo.saltproject.io/salt/py3/ubuntu/20.04/amd64/latest/SALT-PROJECT-GPG-PUBKEY-2023.pub", + aptkey=False, + ) + + ret = _run() + assert "Failed to configure repo" in ret.comment + assert "Could not add key" in ret.comment From fd889285628601678ce719e9feee814b15b852ae Mon Sep 17 00:00:00 2001 From: nicholasmhughes Date: Tue, 23 May 2023 09:56:49 -0400 Subject: [PATCH 056/152] update reqs... again --- requirements/static/ci/py3.10/cloud.txt | 2 +- requirements/static/ci/py3.10/darwin.txt | 4 + requirements/static/ci/py3.10/freebsd.txt | 4 + requirements/static/ci/py3.10/linux.txt | 7 +- .../static/ci/py3.11/darwin-crypto.txt | 10 -- requirements/static/ci/py3.11/darwin.txt | 4 + .../static/ci/py3.11/freebsd-crypto.txt | 10 -- requirements/static/ci/py3.11/freebsd.txt | 4 + .../static/ci/py3.11/linux-crypto.txt | 10 -- requirements/static/ci/py3.11/linux.txt | 7 +- .../static/ci/py3.11/windows-crypto.txt | 10 -- requirements/static/ci/py3.7/cloud.txt | 2 +- requirements/static/ci/py3.7/docs.txt | 17 ++- requirements/static/ci/py3.7/freebsd.txt | 41 +++---- requirements/static/ci/py3.7/lint.txt | 86 +++++++++----- requirements/static/ci/py3.7/linux.txt | 111 +++++++++++------- requirements/static/ci/py3.8/cloud.txt | 2 +- requirements/static/ci/py3.8/docs.txt | 17 ++- requirements/static/ci/py3.8/freebsd.txt | 39 +++--- requirements/static/ci/py3.8/lint.txt | 65 +++++----- requirements/static/ci/py3.8/linux.txt | 94 ++++++++------- requirements/static/ci/py3.9/cloud.txt | 2 +- requirements/static/ci/py3.9/darwin.txt | 4 + requirements/static/ci/py3.9/docs.txt | 27 ++--- requirements/static/ci/py3.9/freebsd.txt | 31 +++-- requirements/static/ci/py3.9/lint.txt | 26 ++-- requirements/static/ci/py3.9/linux.txt | 34 +++--- 27 files changed, 353 insertions(+), 317 deletions(-) delete mode 100644 requirements/static/ci/py3.11/darwin-crypto.txt delete mode 100644 requirements/static/ci/py3.11/freebsd-crypto.txt delete mode 100644 requirements/static/ci/py3.11/linux-crypto.txt delete mode 100644 requirements/static/ci/py3.11/windows-crypto.txt diff --git a/requirements/static/ci/py3.10/cloud.txt b/requirements/static/ci/py3.10/cloud.txt index 1a4b4e1d914..7f268048c6e 100644 --- a/requirements/static/ci/py3.10/cloud.txt +++ b/requirements/static/ci/py3.10/cloud.txt @@ -169,7 +169,7 @@ junos-eznc==2.6.0 ; sys_platform != "win32" and python_version <= "3.10" # via -r requirements/static/ci/common.in jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" +kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index 7118f507b19..56bd799ca22 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -217,6 +217,8 @@ ncclient==0.6.9 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -356,6 +358,7 @@ requests==2.25.1 # kubernetes # moto # pyvmomi + # requests-oauthlib # responses # vcert # vultr @@ -382,6 +385,7 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # junos-eznc # kubernetes diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index 4dce26c95a1..d51925d5a39 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -214,6 +214,8 @@ ncclient==0.6.9 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -355,6 +357,7 @@ requests==2.25.1 # kubernetes # moto # pyvmomi + # requests-oauthlib # responses # vcert responses==0.10.6 @@ -380,6 +383,7 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # junos-eznc # kazoo diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index bf49bfad382..94ff838c9dd 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -225,6 +225,8 @@ ncclient==0.6.9 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==22.0 @@ -276,7 +278,8 @@ pyiface==0.0.11 pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via twilio + # via + # twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -385,6 +388,7 @@ requests==2.25.1 # moto # python-consul # pyvmomi + # requests-oauthlib # responses # twilio # vcert @@ -416,6 +420,7 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # junos-eznc # kazoo diff --git a/requirements/static/ci/py3.11/darwin-crypto.txt b/requirements/static/ci/py3.11/darwin-crypto.txt deleted file mode 100644 index 32d8f607198..00000000000 --- a/requirements/static/ci/py3.11/darwin-crypto.txt +++ /dev/null @@ -1,10 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/static/ci/py3.11/darwin-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.11/darwin.txt' requirements/static/ci/crypto.in -# -m2crypto==0.38.0 - # via -r requirements/static/ci/crypto.in -pycryptodome==3.18.0 - # via -r requirements/static/ci/crypto.in diff --git a/requirements/static/ci/py3.11/darwin.txt b/requirements/static/ci/py3.11/darwin.txt index a74d24b212e..9007f9a4600 100644 --- a/requirements/static/ci/py3.11/darwin.txt +++ b/requirements/static/ci/py3.11/darwin.txt @@ -200,6 +200,8 @@ multidict==6.0.2 # via # aiohttp # yarl +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -327,6 +329,7 @@ requests==2.25.1 # kubernetes # moto # pyvmomi + # requests-oauthlib # responses # vcert # vultr @@ -351,6 +354,7 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # kubernetes # mock diff --git a/requirements/static/ci/py3.11/freebsd-crypto.txt b/requirements/static/ci/py3.11/freebsd-crypto.txt deleted file mode 100644 index 535a2529e8f..00000000000 --- a/requirements/static/ci/py3.11/freebsd-crypto.txt +++ /dev/null @@ -1,10 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/static/ci/py3.11/freebsd-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.11/freebsd.txt' requirements/static/ci/crypto.in -# -m2crypto==0.38.0 - # via -r requirements/static/ci/crypto.in -pycryptodome==3.18.0 - # via -r requirements/static/ci/crypto.in diff --git a/requirements/static/ci/py3.11/freebsd.txt b/requirements/static/ci/py3.11/freebsd.txt index 4d9264a3ec0..fe48d3881da 100644 --- a/requirements/static/ci/py3.11/freebsd.txt +++ b/requirements/static/ci/py3.11/freebsd.txt @@ -201,6 +201,8 @@ multidict==6.0.2 # via # aiohttp # yarl +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -332,6 +334,7 @@ requests==2.25.1 # kubernetes # moto # pyvmomi + # requests-oauthlib # responses # vcert responses==0.10.6 @@ -355,6 +358,7 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # kazoo # kubernetes diff --git a/requirements/static/ci/py3.11/linux-crypto.txt b/requirements/static/ci/py3.11/linux-crypto.txt deleted file mode 100644 index 69646264e97..00000000000 --- a/requirements/static/ci/py3.11/linux-crypto.txt +++ /dev/null @@ -1,10 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/static/ci/py3.11/linux-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.11/linux.txt' requirements/static/ci/crypto.in -# -m2crypto==0.38.0 - # via -r requirements/static/ci/crypto.in -pycryptodome==3.18.0 - # via -r requirements/static/ci/crypto.in diff --git a/requirements/static/ci/py3.11/linux.txt b/requirements/static/ci/py3.11/linux.txt index f6c008d55b1..610aea8f3f2 100644 --- a/requirements/static/ci/py3.11/linux.txt +++ b/requirements/static/ci/py3.11/linux.txt @@ -212,6 +212,8 @@ multidict==6.0.2 # via # aiohttp # yarl +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==22.0 @@ -259,7 +261,8 @@ pyiface==0.0.11 pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via twilio + # via + # twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -362,6 +365,7 @@ requests==2.25.1 # moto # python-consul # pyvmomi + # requests-oauthlib # responses # twilio # vcert @@ -391,6 +395,7 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # kazoo # kubernetes diff --git a/requirements/static/ci/py3.11/windows-crypto.txt b/requirements/static/ci/py3.11/windows-crypto.txt deleted file mode 100644 index 14e0be2eeee..00000000000 --- a/requirements/static/ci/py3.11/windows-crypto.txt +++ /dev/null @@ -1,10 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/static/ci/py3.11/windows-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.11/windows.txt' requirements/static/ci/crypto.in -# -m2crypto==0.38.0 - # via -r requirements/static/ci/crypto.in -pycryptodome==3.18.0 - # via -r requirements/static/ci/crypto.in diff --git a/requirements/static/ci/py3.7/cloud.txt b/requirements/static/ci/py3.7/cloud.txt index 3a4218ca8bf..46a7b994d25 100644 --- a/requirements/static/ci/py3.7/cloud.txt +++ b/requirements/static/ci/py3.7/cloud.txt @@ -193,7 +193,7 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" +kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in diff --git a/requirements/static/ci/py3.7/docs.txt b/requirements/static/ci/py3.7/docs.txt index 6bc786d2c21..7876517ba0e 100644 --- a/requirements/static/ci/py3.7/docs.txt +++ b/requirements/static/ci/py3.7/docs.txt @@ -49,16 +49,23 @@ importlib-metadata==4.6.4 # -c requirements/static/ci/py3.7/linux.txt # sphinxcontrib-spelling jaraco.classes==3.2.1 - # via jaraco.collections + # via + # -c requirements/static/ci/py3.7/linux.txt + # jaraco.collections jaraco.collections==3.4.0 - # via cherrypy + # via + # -c requirements/static/ci/py3.7/linux.txt + # cherrypy jaraco.functools==2.0 # via + # -c requirements/static/ci/py3.7/linux.txt # cheroot # jaraco.text # tempora jaraco.text==3.5.1 - # via jaraco.collections + # via + # -c requirements/static/ci/py3.7/linux.txt + # jaraco.collections jinja2==3.1.2 # via # -c requirements/static/ci/py3.7/linux.txt @@ -191,7 +198,9 @@ urllib3==1.26.6 # -c requirements/static/ci/py3.7/linux.txt # requests zc.lockfile==1.4 - # via cherrypy + # via + # -c requirements/static/ci/py3.7/linux.txt + # cherrypy zipp==3.5.0 # via # -c requirements/static/ci/py3.7/linux.txt diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt index e915fe69d65..f708cbce480 100644 --- a/requirements/static/ci/py3.7/freebsd.txt +++ b/requirements/static/ci/py3.7/freebsd.txt @@ -27,9 +27,7 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -autocommand==2.2.2 - # via jaraco.text -backports.entry-points-selectable==1.2.0 +backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 # via @@ -157,24 +155,20 @@ importlib-metadata==4.6.4 # pluggy # pytest # virtualenv -importlib-resources==5.12.0 - # via jaraco.text -inflect==6.0.4 - # via jaraco.text iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -jaraco.collections==4.1.0 +jaraco.classes==3.2.1 + # via jaraco.collections +jaraco.collections==3.4.0 # via cherrypy -jaraco.context==4.3.0 - # via jaraco.text -jaraco.functools==3.6.0 +jaraco.functools==2.0 # via # cheroot # jaraco.text # tempora -jaraco.text==3.11.1 +jaraco.text==3.5.1 # via jaraco.collections jinja2==3.1.2 # via @@ -196,13 +190,13 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" +kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" +libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt @@ -228,8 +222,8 @@ more-itertools==5.0.0 # via # cheroot # cherrypy + # jaraco.classes # jaraco.functools - # jaraco.text moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -253,6 +247,8 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.0 # via junos-eznc +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -293,12 +289,10 @@ pyasn1==0.4.8 # via # pyasn1-modules # rsa -pycparser==2.21 +pycparser==2.17 # via cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pydantic==1.10.7 - # via inflect pyeapi==0.8.3 # via napalm pygit2==1.8.0 ; python_version >= "3.7" @@ -403,6 +397,7 @@ requests==2.25.1 # moto # napalm # pyvmomi + # requests-oauthlib # responses # vcert responses==0.10.6 @@ -431,6 +426,7 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # junos-eznc # kazoo @@ -471,13 +467,12 @@ tomli==2.0.1 # via pytest transitions==0.8.1 # via junos-eznc -typing-extensions==4.5.0 +typing-extensions==3.10.0.0 # via # aiohttp # async-timeout # gitpython # importlib-metadata - # pydantic # pytest-shell-utilities # pytest-system-statistics # yarl @@ -511,12 +506,10 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==3.0.post1 +zc.lockfile==1.4 # via cherrypy zipp==3.5.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/ci/py3.7/lint.txt b/requirements/static/ci/py3.7/lint.txt index 26c60c2b21a..02408a88048 100644 --- a/requirements/static/ci/py3.7/lint.txt +++ b/requirements/static/ci/py3.7/lint.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: +# This file is autogenerated by pip-compile +# To update, run: # # pip-compile --output-file=requirements/static/ci/py3.7/lint.txt --pip-args='--constraint=requirements/static/ci/py3.7/linux.txt' requirements/base.txt requirements/static/ci/common.in requirements/static/ci/lint.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # @@ -8,10 +8,10 @@ aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 # via aiohttp -ansible==7.5.0 ; python_version >= "3.9" - # via -r requirements/static/ci/linux.in -ansible-core==2.14.6 +ansible-core==2.11.4 # via ansible +ansible==4.4.0 ; python_version < "3.9" + # via -r requirements/static/ci/linux.in apache-libcloud==3.3.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in apscheduler==3.6.3 @@ -24,22 +24,26 @@ astroid==2.3.3 # via pylint async-timeout==4.0.2 # via aiohttp +asynctest==0.13.0 + # via aiohttp attrs==21.2.0 # via # aiohttp # jsonschema -backports-entry-points-selectable==1.1.0 +backports.entry-points-selectable==1.1.0 # via virtualenv +backports.zoneinfo==0.2.1 + # via tzlocal bcrypt==3.2.0 # via # paramiko # passlib -boto==2.49.0 - # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto +boto==2.49.0 + # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -143,22 +147,29 @@ idna==3.2 immutables==0.16 # via contextvars importlib-metadata==4.6.4 - # via -r requirements/static/pkg/linux.in + # via + # -r requirements/static/pkg/linux.in + # backports.entry-points-selectable + # click + # jsonschema + # mako + # moto + # virtualenv ipaddress==1.0.23 # via kubernetes isort==4.3.21 # via pylint -jaraco-classes==3.2.1 - # via jaraco-collections -jaraco-collections==3.4.0 +jaraco.classes==3.2.1 + # via jaraco.collections +jaraco.collections==3.4.0 # via cherrypy -jaraco-functools==3.3.0 +jaraco.functools==3.3.0 # via # cheroot - # jaraco-text + # jaraco.text # tempora -jaraco-text==3.5.1 - # via jaraco-collections +jaraco.text==3.5.1 + # via jaraco.collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -180,12 +191,16 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in lazy-object-proxy==1.4.3 # via astroid +libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in loguru==0.6.0 # via ciscoconfparse looseversion==1.0.2 @@ -214,8 +229,8 @@ more-itertools==8.8.0 # via # cheroot # cherrypy - # jaraco-classes - # jaraco-functools + # jaraco.classes + # jaraco.functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -247,8 +262,9 @@ packaging==21.3 # via # -r requirements/base.txt # ansible-core -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -266,27 +282,26 @@ portend==2.7.1 # via cherrypy psutil==5.8.0 # via -r requirements/base.txt +pyasn1-modules==0.2.8 + # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.8 - # via google-auth pycodestyle==2.5.0 # via saltpylint -pycparser==2.21 ; python_version >= "3.9" - # via - # -r requirements/static/ci/common.in - # -r requirements/static/pkg/linux.in - # cffi +pycparser==2.20 + # via cffi pycryptodomex==3.10.1 # via -r requirements/crypto.txt pyeapi==0.8.4 # via napalm -pygit2==1.12.1 ; python_version > "3.8" +pygit2==1.0.3 ; python_version <= "3.8" # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pyjwt==2.4.0 # via twilio pylint==2.4.4 @@ -348,10 +363,10 @@ pyyaml==5.4.1 # yamlordereddictloader pyzmq==23.2.0 ; python_version < "3.11" # via -r requirements/zeromq.txt -redis==3.5.3 - # via redis-py-cluster redis-py-cluster==2.1.3 # via -r requirements/static/ci/linux.in +redis==3.5.3 + # via redis-py-cluster requests==2.26.0 # via # -r requirements/base.txt @@ -402,6 +417,7 @@ six==1.16.0 # geomet # jsonschema # junos-eznc + # kazoo # kubernetes # ncclient # paramiko @@ -445,6 +461,16 @@ transitions==0.8.8 # via junos-eznc twilio==7.9.2 # via -r requirements/static/ci/linux.in +typed-ast==1.4.1 + # via astroid +typing-extensions==3.10.0.0 + # via + # aiohttp + # async-timeout + # gitpython + # immutables + # importlib-metadata + # yarl tzlocal==3.0 # via apscheduler urllib3==1.26.6 @@ -476,7 +502,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc-lockfile==2.0 +zc.lockfile==2.0 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.7/linux.txt b/requirements/static/ci/py3.7/linux.txt index 3730da11095..01a69bc3d30 100644 --- a/requirements/static/ci/py3.7/linux.txt +++ b/requirements/static/ci/py3.7/linux.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: +# This file is autogenerated by pip-compile +# To update, run: # # pip-compile --output-file=requirements/static/ci/py3.7/linux.txt --pip-args='--constraint=requirements/static/pkg/py3.7/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # @@ -8,10 +8,10 @@ aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 # via aiohttp -ansible==7.5.0 ; python_version >= "3.9" - # via -r requirements/static/ci/linux.in -ansible-core==2.14.6 +ansible-core==2.11.7 # via ansible +ansible==4.4.0 ; python_version < "3.9" + # via -r requirements/static/ci/linux.in apache-libcloud==2.5.0 ; sys_platform != "win32" # via -r requirements/static/ci/common.in apscheduler==3.6.3 @@ -22,6 +22,8 @@ asn1crypto==1.3.0 # oscrypto async-timeout==4.0.2 # via aiohttp +asynctest==0.13.0 + # via aiohttp attrs==20.3.0 # via # aiohttp @@ -31,18 +33,18 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -backports-entry-points-selectable==1.1.0 +backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 # via # paramiko # passlib -boto==2.49.0 - # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto +boto==2.49.0 + # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -154,22 +156,30 @@ idna==2.8 immutables==0.15 # via contextvars importlib-metadata==4.6.4 - # via -r requirements/static/pkg/linux.in + # via + # -r requirements/static/pkg/linux.in + # backports.entry-points-selectable + # jsonschema + # mako + # moto + # pluggy + # pytest + # virtualenv iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -jaraco-classes==3.2.1 - # via jaraco-collections -jaraco-collections==3.4.0 +jaraco.classes==3.2.1 + # via jaraco.collections +jaraco.collections==3.4.0 # via cherrypy -jaraco-functools==2.0 +jaraco.functools==2.0 # via # cheroot - # jaraco-text + # jaraco.text # tempora -jaraco-text==3.5.1 - # via jaraco-collections +jaraco.text==3.5.1 + # via jaraco.collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -191,10 +201,14 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in +libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -219,8 +233,8 @@ more-itertools==5.0.0 # via # cheroot # cherrypy - # jaraco-classes - # jaraco-functools + # jaraco.classes + # jaraco.functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -244,6 +258,8 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.0 # via junos-eznc +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -251,8 +267,9 @@ packaging==21.3 # -r requirements/base.txt # ansible-core # pytest -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -278,27 +295,27 @@ psutil==5.8.0 # pytest-salt-factories # pytest-shell-utilities # pytest-system-statistics +pyasn1-modules==0.2.4 + # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.4 - # via google-auth -pycparser==2.21 ; python_version >= "3.9" - # via - # -r requirements/static/ci/common.in - # -r requirements/static/pkg/linux.in - # cffi +pycparser==2.17 + # via cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt pyeapi==0.8.3 # via napalm -pygit2==1.12.1 ; python_version > "3.8" +pygit2==1.0.3 ; python_version <= "3.8" # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via twilio + # via + # twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -317,18 +334,6 @@ pyserial==3.4 # via # junos-eznc # netmiko -pytest==7.2.0 ; python_version > "3.6" - # via - # -r requirements/pytest.txt - # pytest-custom-exit-code - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-subtests - # pytest-system-statistics - # pytest-tempdir - # pytest-timeout pytest-custom-exit-code==0.3.0 # via -r requirements/pytest.txt pytest-helpers-namespace==2021.4.29 @@ -357,6 +362,18 @@ pytest-tempdir==2019.10.12 # pytest-salt-factories pytest-timeout==1.4.2 # via -r requirements/pytest.txt +pytest==7.2.0 ; python_version > "3.6" + # via + # -r requirements/pytest.txt + # pytest-custom-exit-code + # pytest-helpers-namespace + # pytest-salt-factories + # pytest-shell-utilities + # pytest-skip-markers + # pytest-subtests + # pytest-system-statistics + # pytest-tempdir + # pytest-timeout python-consul==1.1.0 # via -r requirements/static/ci/linux.in python-dateutil==2.8.1 @@ -397,10 +414,10 @@ pyzmq==23.2.0 ; python_version < "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -redis==3.5.3 - # via redis-py-cluster redis-py-cluster==2.1.3 # via -r requirements/static/ci/linux.in +redis==3.5.3 + # via redis-py-cluster requests==2.25.1 # via # -r requirements/base.txt @@ -413,6 +430,7 @@ requests==2.25.1 # napalm # python-consul # pyvmomi + # requests-oauthlib # responses # twilio # vcert @@ -447,8 +465,10 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # junos-eznc + # kazoo # kubernetes # mock # more-itertools @@ -497,8 +517,13 @@ twilio==7.9.2 # via -r requirements/static/ci/linux.in typing-extensions==3.10.0.0 # via + # aiohttp + # async-timeout + # gitpython + # importlib-metadata # pytest-shell-utilities # pytest-system-statistics + # yarl tzlocal==2.1 # via apscheduler urllib3==1.26.6 @@ -531,7 +556,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc-lockfile==1.4 +zc.lockfile==1.4 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.8/cloud.txt b/requirements/static/ci/py3.8/cloud.txt index e1450e08eb4..9927709a5ce 100644 --- a/requirements/static/ci/py3.8/cloud.txt +++ b/requirements/static/ci/py3.8/cloud.txt @@ -182,7 +182,7 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" +kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in diff --git a/requirements/static/ci/py3.8/docs.txt b/requirements/static/ci/py3.8/docs.txt index 5a398c0e569..bd6095559a2 100644 --- a/requirements/static/ci/py3.8/docs.txt +++ b/requirements/static/ci/py3.8/docs.txt @@ -45,16 +45,23 @@ immutables==0.15 # -c requirements/static/ci/py3.8/linux.txt # contextvars jaraco.classes==3.2.1 - # via jaraco.collections + # via + # -c requirements/static/ci/py3.8/linux.txt + # jaraco.collections jaraco.collections==3.4.0 - # via cherrypy + # via + # -c requirements/static/ci/py3.8/linux.txt + # cherrypy jaraco.functools==2.0 # via + # -c requirements/static/ci/py3.8/linux.txt # cheroot # jaraco.text # tempora jaraco.text==3.5.1 - # via jaraco.collections + # via + # -c requirements/static/ci/py3.8/linux.txt + # jaraco.collections jinja2==3.1.2 # via # -c requirements/static/ci/py3.8/linux.txt @@ -181,7 +188,9 @@ urllib3==1.26.6 # -c requirements/static/ci/py3.8/linux.txt # requests zc.lockfile==1.4 - # via cherrypy + # via + # -c requirements/static/ci/py3.8/linux.txt + # cherrypy # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index 86710f8db1a..33cee8d0f1e 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -25,9 +25,7 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -autocommand==2.2.2 - # via jaraco.text -backports.entry-points-selectable==1.2.0 +backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 # via @@ -147,24 +145,20 @@ immutables==0.15 # via contextvars importlib-metadata==4.6.4 # via -r requirements/static/pkg/freebsd.in -importlib-resources==5.12.0 - # via jaraco.text -inflect==6.0.4 - # via jaraco.text iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -jaraco.collections==4.1.0 +jaraco.classes==3.2.1 + # via jaraco.collections +jaraco.collections==3.4.0 # via cherrypy -jaraco.context==4.3.0 - # via jaraco.text -jaraco.functools==3.6.0 +jaraco.functools==2.0 # via # cheroot # jaraco.text # tempora -jaraco.text==3.11.1 +jaraco.text==3.5.1 # via jaraco.collections jinja2==3.1.2 # via @@ -186,13 +180,13 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" +kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" +libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt @@ -218,8 +212,8 @@ more-itertools==5.0.0 # via # cheroot # cherrypy + # jaraco.classes # jaraco.functools - # jaraco.text moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -243,6 +237,8 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -283,12 +279,10 @@ pyasn1==0.4.8 # via # pyasn1-modules # rsa -pycparser==2.21 +pycparser==2.17 # via cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pydantic==1.10.7 - # via inflect pyeapi==0.8.3 # via napalm pygit2==1.8.0 ; python_version >= "3.7" @@ -393,6 +387,7 @@ requests==2.25.1 # moto # napalm # pyvmomi + # requests-oauthlib # responses # vcert responses==0.10.6 @@ -421,6 +416,7 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # junos-eznc # kazoo @@ -463,7 +459,6 @@ transitions==0.8.1 # via junos-eznc typing-extensions==4.2.0 # via - # pydantic # pytest-shell-utilities # pytest-system-statistics urllib3==1.26.6 @@ -496,12 +491,10 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==3.0.post1 +zc.lockfile==1.4 # via cherrypy zipp==3.5.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/ci/py3.8/lint.txt b/requirements/static/ci/py3.8/lint.txt index 007d6224a0a..2d718e2f88a 100644 --- a/requirements/static/ci/py3.8/lint.txt +++ b/requirements/static/ci/py3.8/lint.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: +# This file is autogenerated by pip-compile +# To update, run: # # pip-compile --output-file=requirements/static/ci/py3.8/lint.txt --pip-args='--constraint=requirements/static/ci/py3.8/linux.txt' requirements/base.txt requirements/static/ci/common.in requirements/static/ci/lint.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # @@ -8,10 +8,10 @@ aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 # via aiohttp -ansible==7.5.0 ; python_version >= "3.9" - # via -r requirements/static/ci/linux.in -ansible-core==2.14.6 +ansible-core==2.11.4 # via ansible +ansible==4.4.0 ; python_version < "3.9" + # via -r requirements/static/ci/linux.in apache-libcloud==3.3.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in apscheduler==3.6.3 @@ -28,18 +28,20 @@ attrs==21.2.0 # via # aiohttp # jsonschema -backports-entry-points-selectable==1.1.0 +backports.entry-points-selectable==1.1.0 # via virtualenv +backports.zoneinfo==0.2.1 + # via tzlocal bcrypt==3.2.0 # via # paramiko # passlib -boto==2.49.0 - # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto +boto==2.49.0 + # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -148,17 +150,17 @@ ipaddress==1.0.23 # via kubernetes isort==4.3.21 # via pylint -jaraco-classes==3.2.1 - # via jaraco-collections -jaraco-collections==3.4.0 +jaraco.classes==3.2.1 + # via jaraco.collections +jaraco.collections==3.4.0 # via cherrypy -jaraco-functools==3.3.0 +jaraco.functools==3.3.0 # via # cheroot - # jaraco-text + # jaraco.text # tempora -jaraco-text==3.5.1 - # via jaraco-collections +jaraco.text==3.5.1 + # via jaraco.collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -180,12 +182,16 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in lazy-object-proxy==1.4.3 # via astroid +libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in loguru==0.6.0 # via ciscoconfparse looseversion==1.0.2 @@ -214,8 +220,8 @@ more-itertools==8.8.0 # via # cheroot # cherrypy - # jaraco-classes - # jaraco-functools + # jaraco.classes + # jaraco.functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -247,8 +253,9 @@ packaging==21.3 # via # -r requirements/base.txt # ansible-core -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -266,27 +273,26 @@ portend==2.7.1 # via cherrypy psutil==5.8.0 # via -r requirements/base.txt +pyasn1-modules==0.2.8 + # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.8 - # via google-auth pycodestyle==2.5.0 # via saltpylint -pycparser==2.21 ; python_version >= "3.9" - # via - # -r requirements/static/ci/common.in - # -r requirements/static/pkg/linux.in - # cffi +pycparser==2.20 + # via cffi pycryptodomex==3.10.1 # via -r requirements/crypto.txt pyeapi==0.8.4 # via napalm -pygit2==1.12.1 ; python_version > "3.8" +pygit2==1.0.3 ; python_version <= "3.8" # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pyjwt==2.4.0 # via twilio pylint==2.4.4 @@ -348,10 +354,10 @@ pyyaml==5.4.1 # yamlordereddictloader pyzmq==23.2.0 ; python_version < "3.11" # via -r requirements/zeromq.txt -redis==3.5.3 - # via redis-py-cluster redis-py-cluster==2.1.3 # via -r requirements/static/ci/linux.in +redis==3.5.3 + # via redis-py-cluster requests==2.26.0 # via # -r requirements/base.txt @@ -402,6 +408,7 @@ six==1.16.0 # geomet # jsonschema # junos-eznc + # kazoo # kubernetes # ncclient # paramiko @@ -476,7 +483,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc-lockfile==2.0 +zc.lockfile==2.0 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.8/linux.txt b/requirements/static/ci/py3.8/linux.txt index e4bc957798d..2e3c30affa6 100644 --- a/requirements/static/ci/py3.8/linux.txt +++ b/requirements/static/ci/py3.8/linux.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: +# This file is autogenerated by pip-compile +# To update, run: # # pip-compile --output-file=requirements/static/ci/py3.8/linux.txt --pip-args='--constraint=requirements/static/pkg/py3.8/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt # @@ -8,10 +8,10 @@ aiohttp==3.8.1 # via etcd3-py aiosignal==1.2.0 # via aiohttp -ansible==7.5.0 ; python_version >= "3.9" - # via -r requirements/static/ci/linux.in -ansible-core==2.14.6 +ansible-core==2.11.7 # via ansible +ansible==4.4.0 ; python_version < "3.9" + # via -r requirements/static/ci/linux.in apache-libcloud==2.5.0 ; sys_platform != "win32" # via -r requirements/static/ci/common.in apscheduler==3.6.3 @@ -31,18 +31,18 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -backports-entry-points-selectable==1.1.0 +backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 # via # paramiko # passlib -boto==2.49.0 - # via -r requirements/static/ci/common.in boto3==1.21.46 ; python_version >= "3.6" # via # -r requirements/static/ci/common.in # moto +boto==2.49.0 + # via -r requirements/static/ci/common.in botocore==1.24.46 # via # boto3 @@ -159,17 +159,17 @@ iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -jaraco-classes==3.2.1 - # via jaraco-collections -jaraco-collections==3.4.0 +jaraco.classes==3.2.1 + # via jaraco.collections +jaraco.collections==3.4.0 # via cherrypy -jaraco-functools==2.0 +jaraco.functools==2.0 # via # cheroot - # jaraco-text + # jaraco.text # tempora -jaraco-text==3.5.1 - # via jaraco-collections +jaraco.text==3.5.1 + # via jaraco.collections jinja2==3.1.2 # via # -r requirements/base.txt @@ -191,10 +191,14 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in +kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in +libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin" + # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt lxml==4.9.1 @@ -219,8 +223,8 @@ more-itertools==5.0.0 # via # cheroot # cherrypy - # jaraco-classes - # jaraco-functools + # jaraco.classes + # jaraco.functools moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -244,6 +248,8 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==22.0 @@ -251,8 +257,9 @@ packaging==22.0 # -r requirements/base.txt # ansible-core # pytest -paramiko==2.10.1 +paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin" # via + # -r requirements/static/ci/common.in # junos-eznc # napalm # ncclient @@ -278,27 +285,27 @@ psutil==5.8.0 # pytest-salt-factories # pytest-shell-utilities # pytest-system-statistics +pyasn1-modules==0.2.4 + # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.4 - # via google-auth -pycparser==2.21 ; python_version >= "3.9" - # via - # -r requirements/static/ci/common.in - # -r requirements/static/pkg/linux.in - # cffi +pycparser==2.17 + # via cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt pyeapi==0.8.3 # via napalm -pygit2==1.12.1 ; python_version > "3.8" +pygit2==1.0.3 ; python_version <= "3.8" # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in +pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" + # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via twilio + # via + # twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -315,18 +322,6 @@ pyserial==3.4 # via # junos-eznc # netmiko -pytest==7.2.0 ; python_version > "3.6" - # via - # -r requirements/pytest.txt - # pytest-custom-exit-code - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-subtests - # pytest-system-statistics - # pytest-tempdir - # pytest-timeout pytest-custom-exit-code==0.3.0 # via -r requirements/pytest.txt pytest-helpers-namespace==2021.4.29 @@ -355,6 +350,18 @@ pytest-tempdir==2019.10.12 # pytest-salt-factories pytest-timeout==1.4.2 # via -r requirements/pytest.txt +pytest==7.2.0 ; python_version > "3.6" + # via + # -r requirements/pytest.txt + # pytest-custom-exit-code + # pytest-helpers-namespace + # pytest-salt-factories + # pytest-shell-utilities + # pytest-skip-markers + # pytest-subtests + # pytest-system-statistics + # pytest-tempdir + # pytest-timeout python-consul==1.1.0 # via -r requirements/static/ci/linux.in python-dateutil==2.8.1 @@ -395,10 +402,10 @@ pyzmq==23.2.0 ; python_version < "3.11" # via # -r requirements/zeromq.txt # pytest-salt-factories -redis==3.5.3 - # via redis-py-cluster redis-py-cluster==2.1.3 # via -r requirements/static/ci/linux.in +redis==3.5.3 + # via redis-py-cluster requests==2.25.1 # via # -r requirements/base.txt @@ -411,6 +418,7 @@ requests==2.25.1 # napalm # python-consul # pyvmomi + # requests-oauthlib # responses # twilio # vcert @@ -445,8 +453,10 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # junos-eznc + # kazoo # kubernetes # mock # more-itertools @@ -529,7 +539,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc-lockfile==1.4 +zc.lockfile==1.4 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.9/cloud.txt b/requirements/static/ci/py3.9/cloud.txt index 4ccd49bcdee..98c5d65da00 100644 --- a/requirements/static/ci/py3.9/cloud.txt +++ b/requirements/static/ci/py3.9/cloud.txt @@ -182,7 +182,7 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" +kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index 080fa1d9f99..94e72bb1e92 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -240,6 +240,8 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -389,6 +391,7 @@ requests==2.25.1 # moto # napalm # pyvmomi + # requests-oauthlib # responses # vcert # vultr @@ -418,6 +421,7 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # junos-eznc # kubernetes diff --git a/requirements/static/ci/py3.9/docs.txt b/requirements/static/ci/py3.9/docs.txt index c40924e4b63..3067f1725f7 100644 --- a/requirements/static/ci/py3.9/docs.txt +++ b/requirements/static/ci/py3.9/docs.txt @@ -6,10 +6,6 @@ # alabaster==0.7.12 # via sphinx -autocommand==2.2.2 - # via - # -c requirements/static/ci/py3.9/linux.txt - # jaraco.text babel==2.9.1 # via sphinx certifi==2022.12.7 @@ -52,25 +48,21 @@ importlib-metadata==6.0.0 # via # -c requirements/static/ci/py3.9/linux.txt # sphinx -inflect==6.0.4 +jaraco.classes==3.2.1 # via # -c requirements/static/ci/py3.9/linux.txt - # jaraco.text -jaraco.collections==4.1.0 + # jaraco.collections +jaraco.collections==3.4.0 # via # -c requirements/static/ci/py3.9/linux.txt # cherrypy -jaraco.context==4.3.0 - # via - # -c requirements/static/ci/py3.9/linux.txt - # jaraco.text -jaraco.functools==3.6.0 +jaraco.functools==2.0 # via # -c requirements/static/ci/py3.9/linux.txt # cheroot # jaraco.text # tempora -jaraco.text==3.11.1 +jaraco.text==3.5.1 # via # -c requirements/static/ci/py3.9/linux.txt # jaraco.collections @@ -108,8 +100,8 @@ more-itertools==5.0.0 # -c requirements/static/ci/py3.9/linux.txt # cheroot # cherrypy + # jaraco.classes # jaraco.functools - # jaraco.text msgpack==1.0.2 # via # -c requirements/static/ci/py3.9/linux.txt @@ -133,10 +125,6 @@ pycryptodomex==3.9.8 # via # -c requirements/static/ci/py3.9/linux.txt # -r requirements/crypto.txt -pydantic==1.10.7 - # via - # -c requirements/static/ci/py3.9/linux.txt - # inflect pyenchant==3.2.2 # via sphinxcontrib-spelling pygments==2.14.0 @@ -197,14 +185,13 @@ typing-extensions==4.2.0 # via # -c requirements/static/ci/py3.9/linux.txt # myst-docutils - # pydantic uc-micro-py==1.0.1 # via linkify-it-py urllib3==1.26.6 # via # -c requirements/static/ci/py3.9/linux.txt # requests -zc.lockfile==3.0.post1 +zc.lockfile==1.4 # via # -c requirements/static/ci/py3.9/linux.txt # cherrypy diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index f845df80c58..5f2a46fc88b 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -25,9 +25,7 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -autocommand==2.2.2 - # via jaraco.text -backports.entry-points-selectable==1.2.0 +backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 # via @@ -147,22 +145,20 @@ immutables==0.15 # via contextvars importlib-metadata==6.0.0 # via -r requirements/static/pkg/freebsd.in -inflect==6.0.4 - # via jaraco.text iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -jaraco.collections==4.1.0 +jaraco.classes==3.2.1 + # via jaraco.collections +jaraco.collections==3.4.0 # via cherrypy -jaraco.context==4.3.0 - # via jaraco.text -jaraco.functools==3.6.0 +jaraco.functools==2.0 # via # cheroot # jaraco.text # tempora -jaraco.text==3.11.1 +jaraco.text==3.5.1 # via jaraco.collections jinja2==3.1.2 # via @@ -184,13 +180,13 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" +kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" +libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt @@ -216,8 +212,8 @@ more-itertools==5.0.0 # via # cheroot # cherrypy + # jaraco.classes # jaraco.functools - # jaraco.text moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -241,6 +237,8 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -288,8 +286,6 @@ pycparser==2.21 ; python_version >= "3.9" # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pydantic==1.10.7 - # via inflect pyeapi==0.8.3 # via napalm pygit2==1.8.0 ; python_version >= "3.7" @@ -394,6 +390,7 @@ requests==2.25.1 # moto # napalm # pyvmomi + # requests-oauthlib # responses # vcert responses==0.10.6 @@ -422,6 +419,7 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # junos-eznc # kazoo @@ -464,7 +462,6 @@ transitions==0.8.1 # via junos-eznc typing-extensions==4.2.0 # via - # pydantic # pytest-shell-utilities # pytest-system-statistics urllib3==1.26.6 @@ -497,7 +494,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==3.0.post1 +zc.lockfile==1.4 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.9/lint.txt b/requirements/static/ci/py3.9/lint.txt index 02529e9a73d..8fbae8824de 100644 --- a/requirements/static/ci/py3.9/lint.txt +++ b/requirements/static/ci/py3.9/lint.txt @@ -28,9 +28,7 @@ attrs==21.2.0 # via # aiohttp # jsonschema -autocommand==2.2.2 - # via jaraco.text -backports.entry-points-selectable==1.2.0 +backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.2.0 # via @@ -146,22 +144,20 @@ immutables==0.16 # via contextvars importlib-metadata==6.0.0 # via -r requirements/static/pkg/linux.in -inflect==6.0.4 - # via jaraco.text ipaddress==1.0.23 # via kubernetes isort==4.3.21 # via pylint -jaraco.collections==4.1.0 +jaraco.classes==3.2.1 + # via jaraco.collections +jaraco.collections==3.4.0 # via cherrypy -jaraco.context==4.3.0 - # via jaraco.text -jaraco.functools==3.6.0 +jaraco.functools==3.3.0 # via # cheroot # jaraco.text # tempora -jaraco.text==3.11.1 +jaraco.text==3.5.1 # via jaraco.collections jinja2==3.1.2 # via @@ -184,7 +180,7 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.3 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" +kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in @@ -222,8 +218,8 @@ more-itertools==8.8.0 # via # cheroot # cherrypy + # jaraco.classes # jaraco.functools - # jaraco.text moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -290,8 +286,6 @@ pycparser==2.21 ; python_version >= "3.9" # cffi pycryptodomex==3.10.1 # via -r requirements/crypto.txt -pydantic==1.10.7 - # via inflect pyeapi==0.8.4 # via napalm pygit2==1.6.1 ; python_version > "3.8" @@ -459,8 +453,6 @@ transitions==0.8.8 # via junos-eznc twilio==7.9.2 # via -r requirements/static/ci/linux.in -typing-extensions==4.5.0 - # via pydantic tzlocal==3.0 # via apscheduler urllib3==1.26.6 @@ -492,7 +484,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==3.0.post1 +zc.lockfile==2.0 # via cherrypy zipp==3.5.0 # via importlib-metadata diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index f255b3348a5..dee19551686 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -31,9 +31,7 @@ attrs==20.3.0 # pytest-shell-utilities # pytest-skip-markers # pytest-system-statistics -autocommand==2.2.2 - # via jaraco.text -backports.entry-points-selectable==1.2.0 +backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.1.6 # via @@ -159,22 +157,20 @@ immutables==0.15 # via contextvars importlib-metadata==6.0.0 # via -r requirements/static/pkg/linux.in -inflect==6.0.4 - # via jaraco.text iniconfig==1.0.1 # via pytest ipaddress==1.0.22 # via kubernetes -jaraco.collections==4.1.0 +jaraco.classes==3.2.1 + # via jaraco.collections +jaraco.collections==3.4.0 # via cherrypy -jaraco.context==4.3.0 - # via jaraco.text -jaraco.functools==3.6.0 +jaraco.functools==2.0 # via # cheroot # jaraco.text # tempora -jaraco.text==3.11.1 +jaraco.text==3.5.1 # via jaraco.collections jinja2==3.1.2 # via @@ -197,13 +193,13 @@ junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10" # napalm jxmlease==1.0.1 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -kazoo==2.9.0 ; sys_platform != "win32" and sys_platform != "darwin" +kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in keyring==5.7.1 # via -r requirements/static/ci/common.in kubernetes==3.0.0 # via -r requirements/static/ci/common.in -libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin" +libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin" # via -r requirements/static/ci/common.in looseversion==1.0.2 # via -r requirements/base.txt @@ -229,8 +225,8 @@ more-itertools==5.0.0 # via # cheroot # cherrypy + # jaraco.classes # jaraco.functools - # jaraco.text moto==3.0.1 ; python_version >= "3.6" # via -r requirements/static/ci/common.in msgpack==1.0.2 @@ -254,6 +250,8 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc +oauthlib==3.2.2 + # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==22.0 @@ -302,8 +300,6 @@ pycparser==2.21 ; python_version >= "3.9" # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pydantic==1.10.7 - # via inflect pyeapi==0.8.3 # via napalm pygit2==1.5.0 ; python_version > "3.8" @@ -313,7 +309,8 @@ pyiface==0.0.11 pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via twilio + # via + # twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -426,6 +423,7 @@ requests==2.25.1 # napalm # python-consul # pyvmomi + # requests-oauthlib # responses # twilio # vcert @@ -460,6 +458,7 @@ six==1.16.0 # genshi # geomet # google-auth + # isodate # jsonschema # junos-eznc # kazoo @@ -511,7 +510,6 @@ twilio==7.9.2 # via -r requirements/static/ci/linux.in typing-extensions==4.2.0 # via - # pydantic # pytest-shell-utilities # pytest-system-statistics tzlocal==2.1 @@ -546,7 +544,7 @@ yamlordereddictloader==0.4.0 # via junos-eznc yarl==1.7.2 # via aiohttp -zc.lockfile==3.0.post1 +zc.lockfile==1.4 # via cherrypy zipp==3.5.0 # via importlib-metadata From 5bb33125b87e03c84999933236fc0e129f9398cf Mon Sep 17 00:00:00 2001 From: nicholasmhughes Date: Tue, 23 May 2023 11:03:38 -0400 Subject: [PATCH 057/152] update reqs... again... again --- requirements/static/ci/py3.10/darwin.txt | 4 ---- requirements/static/ci/py3.10/freebsd.txt | 4 ---- requirements/static/ci/py3.10/linux.txt | 7 +------ requirements/static/ci/py3.11/darwin-crypto.txt | 10 ++++++++++ requirements/static/ci/py3.11/darwin.txt | 4 ---- requirements/static/ci/py3.11/freebsd-crypto.txt | 10 ++++++++++ requirements/static/ci/py3.11/freebsd.txt | 4 ---- requirements/static/ci/py3.11/linux-crypto.txt | 10 ++++++++++ requirements/static/ci/py3.11/linux.txt | 7 +------ requirements/static/ci/py3.11/windows-crypto.txt | 10 ++++++++++ requirements/static/ci/py3.7/freebsd.txt | 4 ---- requirements/static/ci/py3.7/linux.txt | 7 +------ requirements/static/ci/py3.8/freebsd.txt | 4 ---- requirements/static/ci/py3.8/linux.txt | 7 +------ requirements/static/ci/py3.9/darwin.txt | 4 ---- requirements/static/ci/py3.9/freebsd.txt | 4 ---- requirements/static/ci/py3.9/linux.txt | 7 +------ 17 files changed, 45 insertions(+), 62 deletions(-) create mode 100644 requirements/static/ci/py3.11/darwin-crypto.txt create mode 100644 requirements/static/ci/py3.11/freebsd-crypto.txt create mode 100644 requirements/static/ci/py3.11/linux-crypto.txt create mode 100644 requirements/static/ci/py3.11/windows-crypto.txt diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index 56bd799ca22..7118f507b19 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -217,8 +217,6 @@ ncclient==0.6.9 # via junos-eznc netaddr==0.7.19 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -358,7 +356,6 @@ requests==2.25.1 # kubernetes # moto # pyvmomi - # requests-oauthlib # responses # vcert # vultr @@ -385,7 +382,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kubernetes diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index d51925d5a39..4dce26c95a1 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -214,8 +214,6 @@ ncclient==0.6.9 # via junos-eznc netaddr==0.7.19 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -357,7 +355,6 @@ requests==2.25.1 # kubernetes # moto # pyvmomi - # requests-oauthlib # responses # vcert responses==0.10.6 @@ -383,7 +380,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kazoo diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index 94ff838c9dd..bf49bfad382 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -225,8 +225,6 @@ ncclient==0.6.9 # via junos-eznc netaddr==0.7.19 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==22.0 @@ -278,8 +276,7 @@ pyiface==0.0.11 pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # twilio + # via twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -388,7 +385,6 @@ requests==2.25.1 # moto # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -420,7 +416,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kazoo diff --git a/requirements/static/ci/py3.11/darwin-crypto.txt b/requirements/static/ci/py3.11/darwin-crypto.txt new file mode 100644 index 00000000000..32d8f607198 --- /dev/null +++ b/requirements/static/ci/py3.11/darwin-crypto.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/static/ci/py3.11/darwin-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.11/darwin.txt' requirements/static/ci/crypto.in +# +m2crypto==0.38.0 + # via -r requirements/static/ci/crypto.in +pycryptodome==3.18.0 + # via -r requirements/static/ci/crypto.in diff --git a/requirements/static/ci/py3.11/darwin.txt b/requirements/static/ci/py3.11/darwin.txt index 9007f9a4600..a74d24b212e 100644 --- a/requirements/static/ci/py3.11/darwin.txt +++ b/requirements/static/ci/py3.11/darwin.txt @@ -200,8 +200,6 @@ multidict==6.0.2 # via # aiohttp # yarl -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -329,7 +327,6 @@ requests==2.25.1 # kubernetes # moto # pyvmomi - # requests-oauthlib # responses # vcert # vultr @@ -354,7 +351,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # kubernetes # mock diff --git a/requirements/static/ci/py3.11/freebsd-crypto.txt b/requirements/static/ci/py3.11/freebsd-crypto.txt new file mode 100644 index 00000000000..535a2529e8f --- /dev/null +++ b/requirements/static/ci/py3.11/freebsd-crypto.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/static/ci/py3.11/freebsd-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.11/freebsd.txt' requirements/static/ci/crypto.in +# +m2crypto==0.38.0 + # via -r requirements/static/ci/crypto.in +pycryptodome==3.18.0 + # via -r requirements/static/ci/crypto.in diff --git a/requirements/static/ci/py3.11/freebsd.txt b/requirements/static/ci/py3.11/freebsd.txt index fe48d3881da..4d9264a3ec0 100644 --- a/requirements/static/ci/py3.11/freebsd.txt +++ b/requirements/static/ci/py3.11/freebsd.txt @@ -201,8 +201,6 @@ multidict==6.0.2 # via # aiohttp # yarl -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -334,7 +332,6 @@ requests==2.25.1 # kubernetes # moto # pyvmomi - # requests-oauthlib # responses # vcert responses==0.10.6 @@ -358,7 +355,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # kazoo # kubernetes diff --git a/requirements/static/ci/py3.11/linux-crypto.txt b/requirements/static/ci/py3.11/linux-crypto.txt new file mode 100644 index 00000000000..69646264e97 --- /dev/null +++ b/requirements/static/ci/py3.11/linux-crypto.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/static/ci/py3.11/linux-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.11/linux.txt' requirements/static/ci/crypto.in +# +m2crypto==0.38.0 + # via -r requirements/static/ci/crypto.in +pycryptodome==3.18.0 + # via -r requirements/static/ci/crypto.in diff --git a/requirements/static/ci/py3.11/linux.txt b/requirements/static/ci/py3.11/linux.txt index 610aea8f3f2..f6c008d55b1 100644 --- a/requirements/static/ci/py3.11/linux.txt +++ b/requirements/static/ci/py3.11/linux.txt @@ -212,8 +212,6 @@ multidict==6.0.2 # via # aiohttp # yarl -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==22.0 @@ -261,8 +259,7 @@ pyiface==0.0.11 pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # twilio + # via twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -365,7 +362,6 @@ requests==2.25.1 # moto # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -395,7 +391,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # kazoo # kubernetes diff --git a/requirements/static/ci/py3.11/windows-crypto.txt b/requirements/static/ci/py3.11/windows-crypto.txt new file mode 100644 index 00000000000..14e0be2eeee --- /dev/null +++ b/requirements/static/ci/py3.11/windows-crypto.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/static/ci/py3.11/windows-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.11/windows.txt' requirements/static/ci/crypto.in +# +m2crypto==0.38.0 + # via -r requirements/static/ci/crypto.in +pycryptodome==3.18.0 + # via -r requirements/static/ci/crypto.in diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt index f708cbce480..d55b15e0433 100644 --- a/requirements/static/ci/py3.7/freebsd.txt +++ b/requirements/static/ci/py3.7/freebsd.txt @@ -247,8 +247,6 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.0 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -397,7 +395,6 @@ requests==2.25.1 # moto # napalm # pyvmomi - # requests-oauthlib # responses # vcert responses==0.10.6 @@ -426,7 +423,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kazoo diff --git a/requirements/static/ci/py3.7/linux.txt b/requirements/static/ci/py3.7/linux.txt index 01a69bc3d30..aaee6da96c1 100644 --- a/requirements/static/ci/py3.7/linux.txt +++ b/requirements/static/ci/py3.7/linux.txt @@ -258,8 +258,6 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.0 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -314,8 +312,7 @@ pyiface==0.0.11 pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # twilio + # via twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -430,7 +427,6 @@ requests==2.25.1 # napalm # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -465,7 +461,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kazoo diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index 33cee8d0f1e..280518b0f0e 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -237,8 +237,6 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -387,7 +385,6 @@ requests==2.25.1 # moto # napalm # pyvmomi - # requests-oauthlib # responses # vcert responses==0.10.6 @@ -416,7 +413,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kazoo diff --git a/requirements/static/ci/py3.8/linux.txt b/requirements/static/ci/py3.8/linux.txt index 2e3c30affa6..eadbee17316 100644 --- a/requirements/static/ci/py3.8/linux.txt +++ b/requirements/static/ci/py3.8/linux.txt @@ -248,8 +248,6 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==22.0 @@ -304,8 +302,7 @@ pyiface==0.0.11 pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # twilio + # via twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -418,7 +415,6 @@ requests==2.25.1 # napalm # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -453,7 +449,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kazoo diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index 94e72bb1e92..080fa1d9f99 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -240,8 +240,6 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -391,7 +389,6 @@ requests==2.25.1 # moto # napalm # pyvmomi - # requests-oauthlib # responses # vcert # vultr @@ -421,7 +418,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kubernetes diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index 5f2a46fc88b..4fcc7bbb720 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -237,8 +237,6 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==21.3 @@ -390,7 +388,6 @@ requests==2.25.1 # moto # napalm # pyvmomi - # requests-oauthlib # responses # vcert responses==0.10.6 @@ -419,7 +416,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kazoo diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index dee19551686..66b35ff18cd 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -250,8 +250,6 @@ netmiko==3.2.0 # via napalm ntc-templates==1.4.1 # via junos-eznc -oauthlib==3.2.2 - # via requests-oauthlib oscrypto==1.2.0 # via certvalidator packaging==22.0 @@ -309,8 +307,7 @@ pyiface==0.0.11 pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in pyjwt==2.4.0 - # via - # twilio + # via twilio pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 @@ -423,7 +420,6 @@ requests==2.25.1 # napalm # python-consul # pyvmomi - # requests-oauthlib # responses # twilio # vcert @@ -458,7 +454,6 @@ six==1.16.0 # genshi # geomet # google-auth - # isodate # jsonschema # junos-eznc # kazoo From bf406808351064085c77d875d31b30a79a12010a Mon Sep 17 00:00:00 2001 From: nicholasmhughes Date: Tue, 23 May 2023 13:25:21 -0400 Subject: [PATCH 058/152] disable unpacking-non-sequence pylint rule --- tests/pytests/unit/states/test_pip.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py index 307ba5e1e65..743561f701b 100644 --- a/tests/pytests/unit/states/test_pip.py +++ b/tests/pytests/unit/states/test_pip.py @@ -67,5 +67,8 @@ def test_issue_64169(caplog): # Confirm that the state continued to install the package as expected. # Only check the 'pkgs' parameter of pip.install - mock_install_call_args, mock_install_call_kwargs = mock_pip_install.call_args + ( + mock_install_call_args, + mock_install_call_kwargs, + ) = mock_pip_install.call_args # pylint: disable=unpacking-non-sequence assert mock_install_call_kwargs["pkgs"] == pkg_to_install From 41fe2c2b396f743ef3a44a708ac54f08a7204c93 Mon Sep 17 00:00:00 2001 From: nicholasmhughes Date: Tue, 23 May 2023 15:10:39 -0400 Subject: [PATCH 059/152] disable unpacking-non-sequence pylint rule --- tests/pytests/unit/states/test_pip.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py index 743561f701b..4624ab25acb 100644 --- a/tests/pytests/unit/states/test_pip.py +++ b/tests/pytests/unit/states/test_pip.py @@ -67,8 +67,10 @@ def test_issue_64169(caplog): # Confirm that the state continued to install the package as expected. # Only check the 'pkgs' parameter of pip.install + # pylint: disable=unpacking-non-sequence ( mock_install_call_args, mock_install_call_kwargs, - ) = mock_pip_install.call_args # pylint: disable=unpacking-non-sequence + ) = mock_pip_install.call_args + # pylint: enable=unpacking-non-sequence assert mock_install_call_kwargs["pkgs"] == pkg_to_install From 79f240c2e974f72f9420b512ba7a5cad65601109 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 13 May 2023 21:28:32 -0700 Subject: [PATCH 060/152] Tornado 6.1 --- requirements/base.txt | 1 + requirements/static/ci/py3.10/darwin.txt | 2 ++ requirements/static/ci/py3.10/freebsd.txt | 2 ++ requirements/static/ci/py3.10/linux.txt | 4 +++- requirements/static/ci/py3.10/windows.txt | 2 ++ requirements/static/ci/py3.11/darwin.txt | 2 ++ requirements/static/ci/py3.11/freebsd.txt | 2 ++ requirements/static/ci/py3.11/linux.txt | 4 +++- requirements/static/ci/py3.11/windows.txt | 2 ++ requirements/static/ci/py3.7/cloud.txt | 2 ++ requirements/static/ci/py3.7/freebsd.txt | 2 ++ requirements/static/ci/py3.7/linux.txt | 4 +++- requirements/static/ci/py3.7/windows.txt | 2 ++ requirements/static/ci/py3.8/freebsd.txt | 2 ++ requirements/static/ci/py3.8/linux.txt | 4 +++- requirements/static/ci/py3.8/windows.txt | 2 ++ requirements/static/ci/py3.9/darwin.txt | 2 ++ requirements/static/ci/py3.9/freebsd.txt | 2 ++ requirements/static/ci/py3.9/linux.txt | 4 +++- requirements/static/ci/py3.9/windows.txt | 2 ++ requirements/static/pkg/py3.10/darwin.txt | 2 ++ requirements/static/pkg/py3.10/freebsd.txt | 2 ++ requirements/static/pkg/py3.10/linux.txt | 2 ++ requirements/static/pkg/py3.10/windows.txt | 2 ++ requirements/static/pkg/py3.11/darwin.txt | 2 ++ requirements/static/pkg/py3.11/freebsd.txt | 2 ++ requirements/static/pkg/py3.11/linux.txt | 2 ++ requirements/static/pkg/py3.11/windows.txt | 2 ++ requirements/static/pkg/py3.7/freebsd.txt | 2 ++ requirements/static/pkg/py3.7/linux.txt | 2 ++ requirements/static/pkg/py3.7/windows.txt | 2 ++ requirements/static/pkg/py3.8/freebsd.txt | 2 ++ requirements/static/pkg/py3.8/linux.txt | 2 ++ requirements/static/pkg/py3.8/windows.txt | 2 ++ requirements/static/pkg/py3.9/darwin.txt | 2 ++ requirements/static/pkg/py3.9/freebsd.txt | 2 ++ requirements/static/pkg/py3.9/linux.txt | 2 ++ requirements/static/pkg/py3.9/windows.txt | 2 ++ salt/__init__.py | 2 +- 39 files changed, 81 insertions(+), 6 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index c19d8804a2b..5cbe319033e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -10,3 +10,4 @@ packaging>=21.3 looseversion # We need contextvars for salt-ssh contextvars +tornado==6.1 diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index d6789f835f7..28bf0ca747b 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -832,6 +832,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1 + # via -r requirements/base.txt transitions==0.8.1 # via junos-eznc typing-extensions==4.2.0 diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index f7c63cdfa59..0f219528a08 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -832,6 +832,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1 + # via -r requirements/base.txt transitions==0.8.1 # via junos-eznc typing-extensions==4.2.0 diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index 271c1edd344..64534cfd48a 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -875,7 +875,9 @@ toml==0.10.2 tomli==2.0.1 # via pytest tornado==6.1 - # via python-telegram-bot + # via + # -r requirements/base.txt + # python-telegram-bot transitions==0.8.1 # via junos-eznc twilio==7.9.2 diff --git a/requirements/static/ci/py3.10/windows.txt b/requirements/static/ci/py3.10/windows.txt index 3f1db8377e8..1dcd739c619 100644 --- a/requirements/static/ci/py3.10/windows.txt +++ b/requirements/static/ci/py3.10/windows.txt @@ -381,6 +381,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1 + # via -r requirements/base.txt typing-extensions==4.4.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.11/darwin.txt b/requirements/static/ci/py3.11/darwin.txt index d9c608388db..5b0e6653f3d 100644 --- a/requirements/static/ci/py3.11/darwin.txt +++ b/requirements/static/ci/py3.11/darwin.txt @@ -794,6 +794,8 @@ timelib==0.2.5 # via -r requirements/darwin.txt toml==0.10.2 # via -r requirements/static/ci/common.in +tornado==6.1 + # via -r requirements/base.txt typing-extensions==4.2.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.11/freebsd.txt b/requirements/static/ci/py3.11/freebsd.txt index 0e45bcde4ad..eabe2ed71bb 100644 --- a/requirements/static/ci/py3.11/freebsd.txt +++ b/requirements/static/ci/py3.11/freebsd.txt @@ -802,6 +802,8 @@ timelib==0.2.5 # via -r requirements/static/pkg/freebsd.in toml==0.10.2 # via -r requirements/static/ci/common.in +tornado==6.1 + # via -r requirements/base.txt typing-extensions==4.2.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.11/linux.txt b/requirements/static/ci/py3.11/linux.txt index 2639d371cc4..65cf9f900cb 100644 --- a/requirements/static/ci/py3.11/linux.txt +++ b/requirements/static/ci/py3.11/linux.txt @@ -845,7 +845,9 @@ timelib==0.2.5 toml==0.10.2 # via -r requirements/static/ci/common.in tornado==6.1 - # via python-telegram-bot + # via + # -r requirements/base.txt + # python-telegram-bot twilio==7.9.2 # via -r requirements/static/ci/linux.in typing-extensions==4.2.0 diff --git a/requirements/static/ci/py3.11/windows.txt b/requirements/static/ci/py3.11/windows.txt index 6c9a0d5e5b7..4499d2a06ec 100644 --- a/requirements/static/ci/py3.11/windows.txt +++ b/requirements/static/ci/py3.11/windows.txt @@ -377,6 +377,8 @@ timelib==0.2.5 # via -r requirements/windows.txt toml==0.10.2 # via -r requirements/static/ci/common.in +tornado==6.1 + # via -r requirements/base.txt typing-extensions==4.4.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.7/cloud.txt b/requirements/static/ci/py3.7/cloud.txt index 857a9d054bc..bdf1ad39fd8 100644 --- a/requirements/static/ci/py3.7/cloud.txt +++ b/requirements/static/ci/py3.7/cloud.txt @@ -906,6 +906,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1 + # via -r requirements/base.txt transitions==0.8.9 # via junos-eznc typing-extensions==3.10.0.0 diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt index 71f214e61b6..05c49e43d4c 100644 --- a/requirements/static/ci/py3.7/freebsd.txt +++ b/requirements/static/ci/py3.7/freebsd.txt @@ -883,6 +883,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1 + # via -r requirements/base.txt transitions==0.8.1 # via junos-eznc typing-extensions==3.10.0.0 diff --git a/requirements/static/ci/py3.7/linux.txt b/requirements/static/ci/py3.7/linux.txt index 9c7a7c86dc3..42a5aecd817 100644 --- a/requirements/static/ci/py3.7/linux.txt +++ b/requirements/static/ci/py3.7/linux.txt @@ -928,7 +928,9 @@ toml==0.10.2 tomli==2.0.1 # via pytest tornado==6.1 - # via python-telegram-bot + # via + # -r requirements/base.txt + # python-telegram-bot transitions==0.8.1 # via junos-eznc twilio==7.9.2 diff --git a/requirements/static/ci/py3.7/windows.txt b/requirements/static/ci/py3.7/windows.txt index b43b83d8678..a78774de098 100644 --- a/requirements/static/ci/py3.7/windows.txt +++ b/requirements/static/ci/py3.7/windows.txt @@ -396,6 +396,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1 + # via -r requirements/base.txt typing-extensions==4.2.0 # via # aiohttp diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index 213c64652d4..f7d3611c958 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -873,6 +873,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1 + # via -r requirements/base.txt transitions==0.8.1 # via junos-eznc typing-extensions==4.2.0 diff --git a/requirements/static/ci/py3.8/linux.txt b/requirements/static/ci/py3.8/linux.txt index e7fd9969871..06db49b21e2 100644 --- a/requirements/static/ci/py3.8/linux.txt +++ b/requirements/static/ci/py3.8/linux.txt @@ -916,7 +916,9 @@ toml==0.10.2 tomli==2.0.1 # via pytest tornado==6.1 - # via python-telegram-bot + # via + # -r requirements/base.txt + # python-telegram-bot transitions==0.8.1 # via junos-eznc twilio==7.9.2 diff --git a/requirements/static/ci/py3.8/windows.txt b/requirements/static/ci/py3.8/windows.txt index 4dca1d45370..2831f738e03 100644 --- a/requirements/static/ci/py3.8/windows.txt +++ b/requirements/static/ci/py3.8/windows.txt @@ -384,6 +384,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1 + # via -r requirements/base.txt typing-extensions==4.2.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index 8be41123871..c2fa345295c 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -876,6 +876,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1 + # via -r requirements/base.txt transitions==0.8.1 # via junos-eznc typing-extensions==4.2.0 diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index 59fa8c6b367..cb3a13c8c03 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -876,6 +876,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1 + # via -r requirements/base.txt transitions==0.8.1 # via junos-eznc typing-extensions==4.2.0 diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index 0821935f033..3c96e9bfa77 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -921,7 +921,9 @@ toml==0.10.2 tomli==2.0.1 # via pytest tornado==6.1 - # via python-telegram-bot + # via + # -r requirements/base.txt + # python-telegram-bot transitions==0.8.1 # via junos-eznc twilio==7.9.2 diff --git a/requirements/static/ci/py3.9/windows.txt b/requirements/static/ci/py3.9/windows.txt index e70f94b0c9b..6174333379a 100644 --- a/requirements/static/ci/py3.9/windows.txt +++ b/requirements/static/ci/py3.9/windows.txt @@ -385,6 +385,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1 + # via -r requirements/base.txt typing-extensions==4.2.0 # via # pytest-shell-utilities diff --git a/requirements/static/pkg/py3.10/darwin.txt b/requirements/static/pkg/py3.10/darwin.txt index eafb49833a4..2e61015f31b 100644 --- a/requirements/static/pkg/py3.10/darwin.txt +++ b/requirements/static/pkg/py3.10/darwin.txt @@ -110,6 +110,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/darwin.txt +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via requests vultr==1.0.1 diff --git a/requirements/static/pkg/py3.10/freebsd.txt b/requirements/static/pkg/py3.10/freebsd.txt index 29f788900b8..dfc5f15fca5 100644 --- a/requirements/static/pkg/py3.10/freebsd.txt +++ b/requirements/static/pkg/py3.10/freebsd.txt @@ -94,6 +94,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/freebsd.in +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via requests zc.lockfile==1.4 diff --git a/requirements/static/pkg/py3.10/linux.txt b/requirements/static/pkg/py3.10/linux.txt index 97a261d561d..4c9e6627e21 100644 --- a/requirements/static/pkg/py3.10/linux.txt +++ b/requirements/static/pkg/py3.10/linux.txt @@ -96,6 +96,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/linux.in +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via requests zc.lockfile==1.4 diff --git a/requirements/static/pkg/py3.10/windows.txt b/requirements/static/pkg/py3.10/windows.txt index 6c8a45998a3..93ce68ef11e 100644 --- a/requirements/static/pkg/py3.10/windows.txt +++ b/requirements/static/pkg/py3.10/windows.txt @@ -124,6 +124,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/windows.txt +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via # -r requirements/windows.txt diff --git a/requirements/static/pkg/py3.11/darwin.txt b/requirements/static/pkg/py3.11/darwin.txt index 0bbc204c682..697fe56dded 100644 --- a/requirements/static/pkg/py3.11/darwin.txt +++ b/requirements/static/pkg/py3.11/darwin.txt @@ -110,6 +110,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/darwin.txt +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via requests vultr==1.0.1 diff --git a/requirements/static/pkg/py3.11/freebsd.txt b/requirements/static/pkg/py3.11/freebsd.txt index a774d5094df..0e063bf599a 100644 --- a/requirements/static/pkg/py3.11/freebsd.txt +++ b/requirements/static/pkg/py3.11/freebsd.txt @@ -94,6 +94,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/freebsd.in +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via requests zc.lockfile==1.4 diff --git a/requirements/static/pkg/py3.11/linux.txt b/requirements/static/pkg/py3.11/linux.txt index f48281b557d..92bac4cd567 100644 --- a/requirements/static/pkg/py3.11/linux.txt +++ b/requirements/static/pkg/py3.11/linux.txt @@ -96,6 +96,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/linux.in +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via requests zc.lockfile==1.4 diff --git a/requirements/static/pkg/py3.11/windows.txt b/requirements/static/pkg/py3.11/windows.txt index b6a308084a4..261994de053 100644 --- a/requirements/static/pkg/py3.11/windows.txt +++ b/requirements/static/pkg/py3.11/windows.txt @@ -124,6 +124,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/windows.txt +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via # -r requirements/windows.txt diff --git a/requirements/static/pkg/py3.7/freebsd.txt b/requirements/static/pkg/py3.7/freebsd.txt index 26e8770955e..357698b2b60 100644 --- a/requirements/static/pkg/py3.7/freebsd.txt +++ b/requirements/static/pkg/py3.7/freebsd.txt @@ -92,6 +92,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/freebsd.in +tornado==6.1 + # via -r requirements/base.txt typing-extensions==3.10.0.0 # via importlib-metadata urllib3==1.26.6 diff --git a/requirements/static/pkg/py3.7/linux.txt b/requirements/static/pkg/py3.7/linux.txt index 43a28df0d00..2eac8ab3d43 100644 --- a/requirements/static/pkg/py3.7/linux.txt +++ b/requirements/static/pkg/py3.7/linux.txt @@ -94,6 +94,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/linux.in +tornado==6.1 + # via -r requirements/base.txt typing-extensions==3.10.0.0 # via importlib-metadata urllib3==1.26.6 diff --git a/requirements/static/pkg/py3.7/windows.txt b/requirements/static/pkg/py3.7/windows.txt index 21965d5b388..d23f409be61 100644 --- a/requirements/static/pkg/py3.7/windows.txt +++ b/requirements/static/pkg/py3.7/windows.txt @@ -125,6 +125,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/windows.txt +tornado==6.1 + # via -r requirements/base.txt typing-extensions==4.4.0 # via # gitpython diff --git a/requirements/static/pkg/py3.8/freebsd.txt b/requirements/static/pkg/py3.8/freebsd.txt index 7abd3ba5be0..b955f4c5608 100644 --- a/requirements/static/pkg/py3.8/freebsd.txt +++ b/requirements/static/pkg/py3.8/freebsd.txt @@ -92,6 +92,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/freebsd.in +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via requests zc.lockfile==1.4 diff --git a/requirements/static/pkg/py3.8/linux.txt b/requirements/static/pkg/py3.8/linux.txt index e48795ddd56..edc86f2abba 100644 --- a/requirements/static/pkg/py3.8/linux.txt +++ b/requirements/static/pkg/py3.8/linux.txt @@ -94,6 +94,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/linux.in +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via requests zc.lockfile==1.4 diff --git a/requirements/static/pkg/py3.8/windows.txt b/requirements/static/pkg/py3.8/windows.txt index 33734387c70..03df1407e8e 100644 --- a/requirements/static/pkg/py3.8/windows.txt +++ b/requirements/static/pkg/py3.8/windows.txt @@ -125,6 +125,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/windows.txt +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via # -r requirements/windows.txt diff --git a/requirements/static/pkg/py3.9/darwin.txt b/requirements/static/pkg/py3.9/darwin.txt index 978ce51a1d3..c9395f2635b 100644 --- a/requirements/static/pkg/py3.9/darwin.txt +++ b/requirements/static/pkg/py3.9/darwin.txt @@ -110,6 +110,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/darwin.txt +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via requests vultr==1.0.1 diff --git a/requirements/static/pkg/py3.9/freebsd.txt b/requirements/static/pkg/py3.9/freebsd.txt index e52705fe720..abf04e7fa91 100644 --- a/requirements/static/pkg/py3.9/freebsd.txt +++ b/requirements/static/pkg/py3.9/freebsd.txt @@ -94,6 +94,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/freebsd.in +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via requests zc.lockfile==1.4 diff --git a/requirements/static/pkg/py3.9/linux.txt b/requirements/static/pkg/py3.9/linux.txt index 4277f66da62..14f6f384a8c 100644 --- a/requirements/static/pkg/py3.9/linux.txt +++ b/requirements/static/pkg/py3.9/linux.txt @@ -96,6 +96,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/linux.in +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via requests zc.lockfile==1.4 diff --git a/requirements/static/pkg/py3.9/windows.txt b/requirements/static/pkg/py3.9/windows.txt index cb76bdf0b3d..d20b63b5708 100644 --- a/requirements/static/pkg/py3.9/windows.txt +++ b/requirements/static/pkg/py3.9/windows.txt @@ -125,6 +125,8 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/windows.txt +tornado==6.1 + # via -r requirements/base.txt urllib3==1.26.6 # via # -r requirements/windows.txt diff --git a/salt/__init__.py b/salt/__init__.py index 6649fdf5683..49fea82bb42 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -12,7 +12,7 @@ if sys.version_info < (3,): ) sys.stderr.flush() -USE_VENDORED_TORNADO = True +USE_VENDORED_TORNADO = False class TornadoImporter: From 5edd2259d8e1b0a708d63529958b12c255672920 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 13 May 2023 21:47:05 -0700 Subject: [PATCH 061/152] Remove stack_context --- salt/client/mixins.py | 43 ++++++++++++++----------------- salt/master.py | 6 +---- salt/minion.py | 60 +++++++++++++++++++------------------------ salt/transport/tcp.py | 1 + 4 files changed, 47 insertions(+), 63 deletions(-) diff --git a/salt/client/mixins.py b/salt/client/mixins.py index 7cdae88ae8a..242755d60ab 100644 --- a/salt/client/mixins.py +++ b/salt/client/mixins.py @@ -14,7 +14,6 @@ from collections.abc import Mapping, MutableMapping import salt._logging import salt.channel.client import salt.exceptions -import salt.ext.tornado.stack_context import salt.minion import salt.output import salt.utils.args @@ -379,29 +378,25 @@ class SyncClientMixin(ClientStateMixin): data["fun_args"] = list(args) + ([kwargs] if kwargs else []) func_globals["__jid_event__"].fire_event(data, "new") - # Initialize a context for executing the method. - with salt.ext.tornado.stack_context.StackContext( - self.functions.context_dict.clone - ): - func = self.functions[fun] - try: - data["return"] = func(*args, **kwargs) - except TypeError as exc: - data[ - "return" - ] = "\nPassed invalid arguments: {}\n\nUsage:\n{}".format( - exc, func.__doc__ - ) - try: - data["success"] = self.context.get("retcode", 0) == 0 - except AttributeError: - # Assume a True result if no context attribute - data["success"] = True - if isinstance(data["return"], dict) and "data" in data["return"]: - # some functions can return boolean values - data["success"] = salt.utils.state.check_result( - data["return"]["data"] - ) + func = self.functions[fun] + try: + data["return"] = func(*args, **kwargs) + except TypeError as exc: + data[ + "return" + ] = "\nPassed invalid arguments: {}\n\nUsage:\n{}".format( + exc, func.__doc__ + ) + try: + data["success"] = self.context.get("retcode", 0) == 0 + except AttributeError: + # Assume a True result if no context attribute + data["success"] = True + if isinstance(data["return"], dict) and "data" in data["return"]: + # some functions can return boolean values + data["success"] = salt.utils.state.check_result( + data["return"]["data"] + ) except (Exception, SystemExit) as ex: # pylint: disable=broad-except if isinstance(ex, salt.exceptions.NotImplemented): data["return"] = str(ex) diff --git a/salt/master.py b/salt/master.py index 9d2239bffbe..6cb5dd178a9 100644 --- a/salt/master.py +++ b/salt/master.py @@ -56,7 +56,6 @@ import salt.utils.zeromq import salt.wheel from salt.config import DEFAULT_INTERVAL from salt.defaults import DEFAULT_TARGET_DELIM -from salt.ext.tornado.stack_context import StackContext from salt.transport import TRANSPORTS from salt.utils.channel import iter_transport_opts from salt.utils.ctx import RequestContext @@ -1105,10 +1104,7 @@ class MWorker(salt.utils.process.SignalHandlingProcess): def run_func(data): return self.aes_funcs.run_func(data["cmd"], data) - with StackContext( - functools.partial(RequestContext, {"data": data, "opts": self.opts}) - ): - ret = run_func(data) + ret = run_func(data) if self.opts["master_stats"]: self._post_stats(start, cmd) diff --git a/salt/minion.py b/salt/minion.py index 3a7c26366fc..10b75fc2bfc 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1691,10 +1691,9 @@ class Minion(MinionBase): timeout_handler = handle_timeout - with salt.ext.tornado.stack_context.ExceptionStackContext(timeout_handler): - # pylint: disable=unexpected-keyword-arg - self._send_req_async(load, timeout, callback=lambda f: None) - # pylint: enable=unexpected-keyword-arg + # pylint: disable=unexpected-keyword-arg + self._send_req_async(load, timeout, callback=lambda f: None) + # pylint: enable=unexpected-keyword-arg return True @salt.ext.tornado.gen.coroutine @@ -1829,11 +1828,7 @@ class Minion(MinionBase): else: return Minion._thread_return(minion_instance, opts, data) - with salt.ext.tornado.stack_context.StackContext( - functools.partial(RequestContext, {"data": data, "opts": opts}) - ): - with salt.ext.tornado.stack_context.StackContext(minion_instance.ctx): - run_func(minion_instance, opts, data) + run_func(minion_instance, opts, data) def _execute_job_function( self, function_name, function_args, executors, opts, data @@ -2253,12 +2248,11 @@ class Minion(MinionBase): timeout_handler() return "" else: - with salt.ext.tornado.stack_context.ExceptionStackContext(timeout_handler): - # pylint: disable=unexpected-keyword-arg - ret_val = self._send_req_async( - load, timeout=timeout, callback=lambda f: None - ) - # pylint: enable=unexpected-keyword-arg + # pylint: disable=unexpected-keyword-arg + ret_val = self._send_req_async( + load, timeout=timeout, callback=lambda f: None + ) + # pylint: enable=unexpected-keyword-arg log.trace("ret_val = %s", ret_val) # pylint: disable=no-member return ret_val @@ -2344,12 +2338,11 @@ class Minion(MinionBase): timeout_handler() return "" else: - with salt.ext.tornado.stack_context.ExceptionStackContext(timeout_handler): - # pylint: disable=unexpected-keyword-arg - ret_val = self._send_req_async( - load, timeout=timeout, callback=lambda f: None - ) - # pylint: enable=unexpected-keyword-arg + # pylint: disable=unexpected-keyword-arg + ret_val = self._send_req_async( + load, timeout=timeout, callback=lambda f: None + ) + # pylint: enable=unexpected-keyword-arg log.trace("ret_val = %s", ret_val) # pylint: disable=no-member return ret_val @@ -3293,19 +3286,18 @@ class Syndic(Minion): log.warning("Unable to forward pub data: %s", args[1]) return True - with salt.ext.tornado.stack_context.ExceptionStackContext(timeout_handler): - self.local.pub_async( - data["tgt"], - data["fun"], - data["arg"], - data["tgt_type"], - data["ret"], - data["jid"], - data["to"], - io_loop=self.io_loop, - callback=lambda _: None, - **kwargs - ) + self.local.pub_async( + data["tgt"], + data["fun"], + data["arg"], + data["tgt_type"], + data["ret"], + data["jid"], + data["to"], + io_loop=self.io_loop, + callback=lambda _: None, + **kwargs + ) def _send_req_sync(self, load, timeout): if self.opts["minion_sign_messages"]: diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index ddde882e764..a28ac19338e 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -363,6 +363,7 @@ class TCPReqServer(salt.transport.base.DaemonizedRequestServer): message_handler: function to call with your payloads """ self.message_handler = message_handler + log.info("ReqServer workers %s", socket) with salt.utils.asynchronous.current_ioloop(io_loop): if USE_LOAD_BALANCER: From a560f7c0f384a07f9cf2a37cfd3c73b0a0e69873 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 13 May 2023 22:03:00 -0700 Subject: [PATCH 062/152] web.asynchronous should be replaced by gen.coroutine --- salt/netapi/rest_tornado/saltnado.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 44c4089ccbb..4fa3611ed2c 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -829,7 +829,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 ret = {"clients": list(self.saltclients.keys()), "return": "Welcome"} self.write(self.serialize(ret)) - @salt.ext.tornado.web.asynchronous + @salt.ext.tornado.gen.coroutine def post(self): # pylint: disable=arguments-differ """ Send one or more Salt commands (lowstates) in the request body @@ -1206,7 +1206,7 @@ class MinionSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 A convenience endpoint for minion related functions """ - @salt.ext.tornado.web.asynchronous + @salt.ext.tornado.gen.coroutine def get(self, mid=None): # pylint: disable=W0221 """ A convenience URL for getting lists of minions or getting minion @@ -1254,7 +1254,7 @@ class MinionSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 self.lowstate = [{"client": "local", "tgt": mid or "*", "fun": "grains.items"}] self.disbatch() - @salt.ext.tornado.web.asynchronous + @salt.ext.tornado.gen.coroutine def post(self): """ Start an execution command and immediately return the job id @@ -1332,7 +1332,7 @@ class JobsSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 A convenience endpoint for job cache data """ - @salt.ext.tornado.web.asynchronous + @salt.ext.tornado.gen.coroutine def get(self, jid=None): # pylint: disable=W0221 """ A convenience URL for getting lists of previously run jobs or getting @@ -1432,7 +1432,7 @@ class RunSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 Endpoint to run commands without normal session handling """ - @salt.ext.tornado.web.asynchronous + @salt.ext.tornado.gen.coroutine def post(self): """ Run commands bypassing the :ref:`normal session handling From 166c07f6dad47978962c0d100674da354a1d6359 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 13 May 2023 22:22:34 -0700 Subject: [PATCH 063/152] Get rid of RequestContext --- requirements/base.txt | 2 +- requirements/static/ci/docs.in | 1 + requirements/static/ci/py3.10/cloud.txt | 2 + requirements/static/ci/py3.10/darwin.txt | 2 +- requirements/static/ci/py3.10/docs.txt | 4 + requirements/static/ci/py3.10/freebsd.txt | 2 +- requirements/static/ci/py3.10/lint.txt | 4 +- .../static/ci/py3.10/pkgtests-windows.txt | 2 + requirements/static/ci/py3.10/pkgtests.txt | 2 + requirements/static/ci/py3.10/windows.txt | 2 +- requirements/static/ci/py3.11/darwin.txt | 2 +- requirements/static/ci/py3.11/freebsd.txt | 2 +- requirements/static/ci/py3.11/windows.txt | 2 +- requirements/static/ci/py3.7/cloud.txt | 2 +- requirements/static/ci/py3.7/docs.txt | 4 + requirements/static/ci/py3.7/freebsd.txt | 2 +- requirements/static/ci/py3.7/lint.txt | 4 +- requirements/static/ci/py3.7/windows.txt | 2 +- requirements/static/ci/py3.8/cloud.txt | 2 + requirements/static/ci/py3.8/docs.txt | 4 + requirements/static/ci/py3.8/freebsd.txt | 2 +- requirements/static/ci/py3.8/lint.txt | 4 +- requirements/static/ci/py3.8/windows.txt | 2 +- requirements/static/ci/py3.9/cloud.txt | 2 + requirements/static/ci/py3.9/darwin.txt | 2 +- requirements/static/ci/py3.9/docs.txt | 4 + requirements/static/ci/py3.9/freebsd.txt | 2 +- requirements/static/ci/py3.9/lint.txt | 4 +- requirements/static/ci/py3.9/windows.txt | 2 +- requirements/static/pkg/py3.10/darwin.txt | 2 +- requirements/static/pkg/py3.10/freebsd.txt | 2 +- requirements/static/pkg/py3.10/linux.txt | 81 ++++++++++--------- requirements/static/pkg/py3.10/windows.txt | 2 +- requirements/static/pkg/py3.11/darwin.txt | 2 +- requirements/static/pkg/py3.11/freebsd.txt | 2 +- requirements/static/pkg/py3.11/linux.txt | 2 +- requirements/static/pkg/py3.11/windows.txt | 2 +- requirements/static/pkg/py3.7/freebsd.txt | 2 +- requirements/static/pkg/py3.7/linux.txt | 2 +- requirements/static/pkg/py3.7/windows.txt | 2 +- requirements/static/pkg/py3.8/freebsd.txt | 2 +- requirements/static/pkg/py3.8/linux.txt | 2 +- requirements/static/pkg/py3.8/windows.txt | 2 +- requirements/static/pkg/py3.9/darwin.txt | 2 +- requirements/static/pkg/py3.9/freebsd.txt | 2 +- requirements/static/pkg/py3.9/linux.txt | 2 +- requirements/static/pkg/py3.9/windows.txt | 2 +- salt/_logging/impl.py | 8 +- salt/master.py | 2 - salt/minion.py | 8 +- 50 files changed, 124 insertions(+), 82 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 5cbe319033e..f56998a7a7f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -10,4 +10,4 @@ packaging>=21.3 looseversion # We need contextvars for salt-ssh contextvars -tornado==6.1 +tornado>=6.1 diff --git a/requirements/static/ci/docs.in b/requirements/static/ci/docs.in index 379223dfb95..f2c9f013b7e 100644 --- a/requirements/static/ci/docs.in +++ b/requirements/static/ci/docs.in @@ -1,5 +1,6 @@ --constraint=./py{py_version}/{platform}.txt + sphinx>=3.5.1; python_version < '3.9' sphinx>=6.1.0; python_version >= '3.9' myst-docutils[linkify] diff --git a/requirements/static/ci/py3.10/cloud.txt b/requirements/static/ci/py3.10/cloud.txt index 4efb5c1129b..705c2f9bd3e 100644 --- a/requirements/static/ci/py3.10/cloud.txt +++ b/requirements/static/ci/py3.10/cloud.txt @@ -850,6 +850,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1.0 + # via -r requirements/base.txt transitions==0.8.9 # via junos-eznc typing-extensions==4.2.0 diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index 28bf0ca747b..14176d821a1 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -832,7 +832,7 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt transitions==0.8.1 # via junos-eznc diff --git a/requirements/static/ci/py3.10/docs.txt b/requirements/static/ci/py3.10/docs.txt index d5432bfb013..3d955fbcf70 100644 --- a/requirements/static/ci/py3.10/docs.txt +++ b/requirements/static/ci/py3.10/docs.txt @@ -177,6 +177,10 @@ tempora==4.1.1 # via # -c requirements/static/ci/py3.10/linux.txt # portend +tornado==6.1.0 + # via + # -c requirements/static/ci/py3.10/linux.txt + # -r requirements/base.txt typing-extensions==4.2.0 # via # -c requirements/static/ci/py3.10/linux.txt diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index 0f219528a08..777cc246220 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -832,7 +832,7 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt transitions==0.8.1 # via junos-eznc diff --git a/requirements/static/ci/py3.10/lint.txt b/requirements/static/ci/py3.10/lint.txt index 5ca4fd978cb..0fe90217d18 100644 --- a/requirements/static/ci/py3.10/lint.txt +++ b/requirements/static/ci/py3.10/lint.txt @@ -821,7 +821,9 @@ toml==0.10.2 # -r requirements/static/ci/common.in # -r requirements/static/ci/lint.in tornado==6.1 - # via python-telegram-bot + # via + # -r requirements/base.txt + # python-telegram-bot transitions==0.8.8 # via junos-eznc twilio==7.9.2 diff --git a/requirements/static/ci/py3.10/pkgtests-windows.txt b/requirements/static/ci/py3.10/pkgtests-windows.txt index 2ba612f0a16..4b4797d9c68 100644 --- a/requirements/static/ci/py3.10/pkgtests-windows.txt +++ b/requirements/static/ci/py3.10/pkgtests-windows.txt @@ -150,6 +150,8 @@ tempora==5.2.1 # via portend tomli==2.0.1 # via pytest +tornado==6.1.0 + # via -r requirements/base.txt typing-extensions==4.4.0 # via # pydantic diff --git a/requirements/static/ci/py3.10/pkgtests.txt b/requirements/static/ci/py3.10/pkgtests.txt index e5f6a8d0499..679c1aa2c76 100644 --- a/requirements/static/ci/py3.10/pkgtests.txt +++ b/requirements/static/ci/py3.10/pkgtests.txt @@ -139,6 +139,8 @@ tempora==5.2.0 # via portend tomli==2.0.1 # via pytest +tornado==6.1.0 + # via -r requirements/base.txt typing-extensions==4.4.0 # via # pydantic diff --git a/requirements/static/ci/py3.10/windows.txt b/requirements/static/ci/py3.10/windows.txt index 1dcd739c619..c6cb3771fa9 100644 --- a/requirements/static/ci/py3.10/windows.txt +++ b/requirements/static/ci/py3.10/windows.txt @@ -381,7 +381,7 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt typing-extensions==4.4.0 # via diff --git a/requirements/static/ci/py3.11/darwin.txt b/requirements/static/ci/py3.11/darwin.txt index 5b0e6653f3d..7a4c059f660 100644 --- a/requirements/static/ci/py3.11/darwin.txt +++ b/requirements/static/ci/py3.11/darwin.txt @@ -794,7 +794,7 @@ timelib==0.2.5 # via -r requirements/darwin.txt toml==0.10.2 # via -r requirements/static/ci/common.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt typing-extensions==4.2.0 # via diff --git a/requirements/static/ci/py3.11/freebsd.txt b/requirements/static/ci/py3.11/freebsd.txt index eabe2ed71bb..ca4b7803414 100644 --- a/requirements/static/ci/py3.11/freebsd.txt +++ b/requirements/static/ci/py3.11/freebsd.txt @@ -802,7 +802,7 @@ timelib==0.2.5 # via -r requirements/static/pkg/freebsd.in toml==0.10.2 # via -r requirements/static/ci/common.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt typing-extensions==4.2.0 # via diff --git a/requirements/static/ci/py3.11/windows.txt b/requirements/static/ci/py3.11/windows.txt index 4499d2a06ec..f731b5934e0 100644 --- a/requirements/static/ci/py3.11/windows.txt +++ b/requirements/static/ci/py3.11/windows.txt @@ -377,7 +377,7 @@ timelib==0.2.5 # via -r requirements/windows.txt toml==0.10.2 # via -r requirements/static/ci/common.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt typing-extensions==4.4.0 # via diff --git a/requirements/static/ci/py3.7/cloud.txt b/requirements/static/ci/py3.7/cloud.txt index bdf1ad39fd8..32561f5fc98 100644 --- a/requirements/static/ci/py3.7/cloud.txt +++ b/requirements/static/ci/py3.7/cloud.txt @@ -906,7 +906,7 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt transitions==0.8.9 # via junos-eznc diff --git a/requirements/static/ci/py3.7/docs.txt b/requirements/static/ci/py3.7/docs.txt index 7876517ba0e..4e2b3faa8e3 100644 --- a/requirements/static/ci/py3.7/docs.txt +++ b/requirements/static/ci/py3.7/docs.txt @@ -185,6 +185,10 @@ tempora==4.1.1 # via # -c requirements/static/ci/py3.7/linux.txt # portend +tornado==6.1.0 + # via + # -c requirements/static/ci/py3.7/linux.txt + # -r requirements/base.txt typing-extensions==3.10.0.0 # via # -c requirements/static/ci/py3.7/linux.txt diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt index 05c49e43d4c..55f03c45788 100644 --- a/requirements/static/ci/py3.7/freebsd.txt +++ b/requirements/static/ci/py3.7/freebsd.txt @@ -883,7 +883,7 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt transitions==0.8.1 # via junos-eznc diff --git a/requirements/static/ci/py3.7/lint.txt b/requirements/static/ci/py3.7/lint.txt index f9dba2dd5ec..28dcdd07a32 100644 --- a/requirements/static/ci/py3.7/lint.txt +++ b/requirements/static/ci/py3.7/lint.txt @@ -878,7 +878,9 @@ toml==0.10.2 # -r requirements/static/ci/common.in # -r requirements/static/ci/lint.in tornado==6.1 - # via python-telegram-bot + # via + # -r requirements/base.txt + # python-telegram-bot transitions==0.8.8 # via junos-eznc twilio==7.9.2 diff --git a/requirements/static/ci/py3.7/windows.txt b/requirements/static/ci/py3.7/windows.txt index a78774de098..ef53e435808 100644 --- a/requirements/static/ci/py3.7/windows.txt +++ b/requirements/static/ci/py3.7/windows.txt @@ -396,7 +396,7 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt typing-extensions==4.2.0 # via diff --git a/requirements/static/ci/py3.8/cloud.txt b/requirements/static/ci/py3.8/cloud.txt index 9fa61127c8f..8e4137889d6 100644 --- a/requirements/static/ci/py3.8/cloud.txt +++ b/requirements/static/ci/py3.8/cloud.txt @@ -895,6 +895,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1.0 + # via -r requirements/base.txt transitions==0.8.9 # via junos-eznc typing-extensions==3.10.0.2 diff --git a/requirements/static/ci/py3.8/docs.txt b/requirements/static/ci/py3.8/docs.txt index bd6095559a2..7117ac0cc9b 100644 --- a/requirements/static/ci/py3.8/docs.txt +++ b/requirements/static/ci/py3.8/docs.txt @@ -177,6 +177,10 @@ tempora==4.1.1 # via # -c requirements/static/ci/py3.8/linux.txt # portend +tornado==6.1.0 + # via + # -c requirements/static/ci/py3.8/linux.txt + # -r requirements/base.txt typing-extensions==4.2.0 # via # -c requirements/static/ci/py3.8/linux.txt diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index f7d3611c958..b342c54f3da 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -873,7 +873,7 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt transitions==0.8.1 # via junos-eznc diff --git a/requirements/static/ci/py3.8/lint.txt b/requirements/static/ci/py3.8/lint.txt index 27832bd6b76..bf84e81e184 100644 --- a/requirements/static/ci/py3.8/lint.txt +++ b/requirements/static/ci/py3.8/lint.txt @@ -869,7 +869,9 @@ toml==0.10.2 # -r requirements/static/ci/common.in # -r requirements/static/ci/lint.in tornado==6.1 - # via python-telegram-bot + # via + # -r requirements/base.txt + # python-telegram-bot transitions==0.8.8 # via junos-eznc twilio==7.9.2 diff --git a/requirements/static/ci/py3.8/windows.txt b/requirements/static/ci/py3.8/windows.txt index 2831f738e03..25e4ce3c70c 100644 --- a/requirements/static/ci/py3.8/windows.txt +++ b/requirements/static/ci/py3.8/windows.txt @@ -384,7 +384,7 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt typing-extensions==4.2.0 # via diff --git a/requirements/static/ci/py3.9/cloud.txt b/requirements/static/ci/py3.9/cloud.txt index fc91bb9d9ad..23caf2ebf87 100644 --- a/requirements/static/ci/py3.9/cloud.txt +++ b/requirements/static/ci/py3.9/cloud.txt @@ -898,6 +898,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +tornado==6.1.0 + # via -r requirements/base.txt transitions==0.8.9 # via junos-eznc typing-extensions==3.10.0.2 diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index c2fa345295c..9075f7fc733 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -876,7 +876,7 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt transitions==0.8.1 # via junos-eznc diff --git a/requirements/static/ci/py3.9/docs.txt b/requirements/static/ci/py3.9/docs.txt index 3067f1725f7..8d0636aafdb 100644 --- a/requirements/static/ci/py3.9/docs.txt +++ b/requirements/static/ci/py3.9/docs.txt @@ -181,6 +181,10 @@ tempora==4.1.1 # via # -c requirements/static/ci/py3.9/linux.txt # portend +tornado==6.1 + # via + # -c requirements/static/ci/py3.9/linux.txt + # -r requirements/base.txt typing-extensions==4.2.0 # via # -c requirements/static/ci/py3.9/linux.txt diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index cb3a13c8c03..108e161f109 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -876,7 +876,7 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt transitions==0.8.1 # via junos-eznc diff --git a/requirements/static/ci/py3.9/lint.txt b/requirements/static/ci/py3.9/lint.txt index 51cf5153aed..787cead5000 100644 --- a/requirements/static/ci/py3.9/lint.txt +++ b/requirements/static/ci/py3.9/lint.txt @@ -870,7 +870,9 @@ toml==0.10.2 # -r requirements/static/ci/common.in # -r requirements/static/ci/lint.in tornado==6.1 - # via python-telegram-bot + # via + # -r requirements/base.txt + # python-telegram-bot transitions==0.8.8 # via junos-eznc twilio==7.9.2 diff --git a/requirements/static/ci/py3.9/windows.txt b/requirements/static/ci/py3.9/windows.txt index 6174333379a..3e29250743d 100644 --- a/requirements/static/ci/py3.9/windows.txt +++ b/requirements/static/ci/py3.9/windows.txt @@ -385,7 +385,7 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt typing-extensions==4.2.0 # via diff --git a/requirements/static/pkg/py3.10/darwin.txt b/requirements/static/pkg/py3.10/darwin.txt index 2e61015f31b..8226cfee877 100644 --- a/requirements/static/pkg/py3.10/darwin.txt +++ b/requirements/static/pkg/py3.10/darwin.txt @@ -110,7 +110,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/darwin.txt -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via requests diff --git a/requirements/static/pkg/py3.10/freebsd.txt b/requirements/static/pkg/py3.10/freebsd.txt index dfc5f15fca5..857f79b4137 100644 --- a/requirements/static/pkg/py3.10/freebsd.txt +++ b/requirements/static/pkg/py3.10/freebsd.txt @@ -94,7 +94,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/freebsd.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via requests diff --git a/requirements/static/pkg/py3.10/linux.txt b/requirements/static/pkg/py3.10/linux.txt index 4c9e6627e21..c11b41ad512 100644 --- a/requirements/static/pkg/py3.10/linux.txt +++ b/requirements/static/pkg/py3.10/linux.txt @@ -4,84 +4,90 @@ # # pip-compile --output-file=requirements/static/pkg/py3.10/linux.txt requirements/base.txt requirements/static/pkg/linux.in requirements/zeromq.txt # -certifi==2022.12.7 +autocommand==2.2.2 + # via jaraco.text +certifi==2023.5.7 # via requests -cffi==1.14.6 +cffi==1.15.1 # via cryptography -chardet==3.0.4 +charset-normalizer==3.1.0 # via requests -cheroot==8.5.2 +cheroot==9.0.0 # via cherrypy -cherrypy==18.6.1 +cherrypy==18.8.0 # via -r requirements/static/pkg/linux.in contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==40.0.2 # via # -r requirements/static/pkg/linux.in # pyopenssl -distro==1.5.0 +distro==1.8.0 # via -r requirements/base.txt -idna==2.8 +idna==3.4 # via requests -immutables==0.15 +immutables==0.19 # via contextvars -importlib-metadata==6.0.0 +importlib-metadata==6.6.0 # via -r requirements/static/pkg/linux.in -jaraco.classes==3.2.1 - # via jaraco.collections -jaraco.collections==3.4.0 +inflect==6.0.4 + # via jaraco.text +jaraco.collections==4.1.0 # via cherrypy -jaraco.functools==2.0 +jaraco.context==4.3.0 + # via jaraco.text +jaraco.functools==3.6.0 # via # cheroot # jaraco.text # tempora -jaraco.text==3.5.1 +jaraco.text==3.11.1 # via jaraco.collections jinja2==3.1.2 # via -r requirements/base.txt jmespath==1.0.1 # via -r requirements/base.txt -looseversion==1.0.2 +looseversion==1.1.2 # via -r requirements/base.txt markupsafe==2.1.2 # via # -r requirements/base.txt # jinja2 -more-itertools==5.0.0 +more-itertools==9.1.0 # via # cheroot # cherrypy - # jaraco.classes # jaraco.functools -msgpack==1.0.2 + # jaraco.text +msgpack==1.0.5 # via -r requirements/base.txt -packaging==22.0 +packaging==23.1 # via -r requirements/base.txt -portend==2.4 +portend==3.1.0 # via cherrypy -psutil==5.8.0 +psutil==5.9.5 # via -r requirements/base.txt pycparser==2.21 ; python_version >= "3.9" # via # -r requirements/static/pkg/linux.in # cffi -pycryptodomex==3.9.8 +pycryptodomex==3.17 # via -r requirements/crypto.txt -pyopenssl==23.0.0 +pydantic==1.10.7 + # via inflect +pyopenssl==23.1.1 # via -r requirements/static/pkg/linux.in -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via -r requirements/static/pkg/linux.in -python-gnupg==0.4.8 +python-gnupg==0.5.0 # via -r requirements/static/pkg/linux.in -pytz==2022.1 +pytz==2023.3 # via tempora -pyyaml==5.4.1 +pyyaml==6.0 # via -r requirements/base.txt -pyzmq==23.2.0 ; python_version < "3.11" +pyzmq==25.0.2 ; python_version < "3.11" # via -r requirements/zeromq.txt -requests==2.25.1 +requests==2.30.0 # via -r requirements/base.txt rpm-vercmp==0.1.2 # via -r requirements/static/pkg/linux.in @@ -90,19 +96,20 @@ setproctitle==1.3.2 six==1.16.0 # via # cheroot - # more-itertools # python-dateutil -tempora==4.1.1 +tempora==5.2.2 # via portend -timelib==0.2.5 +timelib==0.3.0 # via -r requirements/static/pkg/linux.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt -urllib3==1.26.6 +typing-extensions==4.5.0 + # via pydantic +urllib3==2.0.2 # via requests -zc.lockfile==1.4 +zc.lockfile==3.0.post1 # via cherrypy -zipp==3.6.0 +zipp==3.15.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/pkg/py3.10/windows.txt b/requirements/static/pkg/py3.10/windows.txt index 93ce68ef11e..7177dd4b114 100644 --- a/requirements/static/pkg/py3.10/windows.txt +++ b/requirements/static/pkg/py3.10/windows.txt @@ -124,7 +124,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/windows.txt -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via diff --git a/requirements/static/pkg/py3.11/darwin.txt b/requirements/static/pkg/py3.11/darwin.txt index 697fe56dded..8bdc136484c 100644 --- a/requirements/static/pkg/py3.11/darwin.txt +++ b/requirements/static/pkg/py3.11/darwin.txt @@ -110,7 +110,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/darwin.txt -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via requests diff --git a/requirements/static/pkg/py3.11/freebsd.txt b/requirements/static/pkg/py3.11/freebsd.txt index 0e063bf599a..88df25bdbe4 100644 --- a/requirements/static/pkg/py3.11/freebsd.txt +++ b/requirements/static/pkg/py3.11/freebsd.txt @@ -94,7 +94,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/freebsd.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via requests diff --git a/requirements/static/pkg/py3.11/linux.txt b/requirements/static/pkg/py3.11/linux.txt index 92bac4cd567..59e4d28a574 100644 --- a/requirements/static/pkg/py3.11/linux.txt +++ b/requirements/static/pkg/py3.11/linux.txt @@ -96,7 +96,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/linux.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via requests diff --git a/requirements/static/pkg/py3.11/windows.txt b/requirements/static/pkg/py3.11/windows.txt index 261994de053..8abb7a87f14 100644 --- a/requirements/static/pkg/py3.11/windows.txt +++ b/requirements/static/pkg/py3.11/windows.txt @@ -124,7 +124,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/windows.txt -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via diff --git a/requirements/static/pkg/py3.7/freebsd.txt b/requirements/static/pkg/py3.7/freebsd.txt index 357698b2b60..80ab3785df0 100644 --- a/requirements/static/pkg/py3.7/freebsd.txt +++ b/requirements/static/pkg/py3.7/freebsd.txt @@ -92,7 +92,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/freebsd.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt typing-extensions==3.10.0.0 # via importlib-metadata diff --git a/requirements/static/pkg/py3.7/linux.txt b/requirements/static/pkg/py3.7/linux.txt index 2eac8ab3d43..dc294914ad0 100644 --- a/requirements/static/pkg/py3.7/linux.txt +++ b/requirements/static/pkg/py3.7/linux.txt @@ -94,7 +94,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/linux.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt typing-extensions==3.10.0.0 # via importlib-metadata diff --git a/requirements/static/pkg/py3.7/windows.txt b/requirements/static/pkg/py3.7/windows.txt index d23f409be61..4c0b3038378 100644 --- a/requirements/static/pkg/py3.7/windows.txt +++ b/requirements/static/pkg/py3.7/windows.txt @@ -125,7 +125,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/windows.txt -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt typing-extensions==4.4.0 # via diff --git a/requirements/static/pkg/py3.8/freebsd.txt b/requirements/static/pkg/py3.8/freebsd.txt index b955f4c5608..1ff0c47d970 100644 --- a/requirements/static/pkg/py3.8/freebsd.txt +++ b/requirements/static/pkg/py3.8/freebsd.txt @@ -92,7 +92,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/freebsd.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via requests diff --git a/requirements/static/pkg/py3.8/linux.txt b/requirements/static/pkg/py3.8/linux.txt index edc86f2abba..4a7a1e20dfe 100644 --- a/requirements/static/pkg/py3.8/linux.txt +++ b/requirements/static/pkg/py3.8/linux.txt @@ -94,7 +94,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/linux.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via requests diff --git a/requirements/static/pkg/py3.8/windows.txt b/requirements/static/pkg/py3.8/windows.txt index 03df1407e8e..3e4a67eaecf 100644 --- a/requirements/static/pkg/py3.8/windows.txt +++ b/requirements/static/pkg/py3.8/windows.txt @@ -125,7 +125,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/windows.txt -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via diff --git a/requirements/static/pkg/py3.9/darwin.txt b/requirements/static/pkg/py3.9/darwin.txt index c9395f2635b..ae6e66a71dd 100644 --- a/requirements/static/pkg/py3.9/darwin.txt +++ b/requirements/static/pkg/py3.9/darwin.txt @@ -110,7 +110,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/darwin.txt -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via requests diff --git a/requirements/static/pkg/py3.9/freebsd.txt b/requirements/static/pkg/py3.9/freebsd.txt index abf04e7fa91..497354a6e09 100644 --- a/requirements/static/pkg/py3.9/freebsd.txt +++ b/requirements/static/pkg/py3.9/freebsd.txt @@ -94,7 +94,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/freebsd.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via requests diff --git a/requirements/static/pkg/py3.9/linux.txt b/requirements/static/pkg/py3.9/linux.txt index 14f6f384a8c..4b72f0b27d5 100644 --- a/requirements/static/pkg/py3.9/linux.txt +++ b/requirements/static/pkg/py3.9/linux.txt @@ -96,7 +96,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/static/pkg/linux.in -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via requests diff --git a/requirements/static/pkg/py3.9/windows.txt b/requirements/static/pkg/py3.9/windows.txt index d20b63b5708..d22409f1fea 100644 --- a/requirements/static/pkg/py3.9/windows.txt +++ b/requirements/static/pkg/py3.9/windows.txt @@ -125,7 +125,7 @@ tempora==4.1.1 # via portend timelib==0.2.5 # via -r requirements/windows.txt -tornado==6.1 +tornado==6.1.0 # via -r requirements/base.txt urllib3==1.26.6 # via diff --git a/salt/_logging/impl.py b/salt/_logging/impl.py index cc18f49a9e9..3656d832048 100644 --- a/salt/_logging/impl.py +++ b/salt/_logging/impl.py @@ -32,7 +32,6 @@ from salt._logging.handlers import SysLogHandler # isort:skip from salt._logging.handlers import WatchedFileHandler # isort:skip from salt._logging.mixins import LoggingMixinMeta # isort:skip from salt.exceptions import LoggingRuntimeError # isort:skip -from salt.utils.ctx import RequestContext # isort:skip from salt.utils.immutabletypes import freeze, ImmutableDict # isort:skip from salt.utils.textformat import TextFormat # isort:skip @@ -238,8 +237,11 @@ class SaltLoggingClass(LOGGING_LOGGER_CLASS, metaclass=LoggingMixinMeta): extra = {} # pylint: disable=no-member - current_jid = RequestContext.current.get("data", {}).get("jid", None) - log_fmt_jid = RequestContext.current.get("opts", {}).get("log_fmt_jid", None) + # XXX TODO + # current_jid = RequestContext.current.get("data", {}).get("jid", None) + # log_fmt_jid = RequestContext.current.get("opts", {}).get("log_fmt_jid", None) + current_jid = "" + log_fmt_jid = "" # pylint: enable=no-member if current_jid is not None: diff --git a/salt/master.py b/salt/master.py index 6cb5dd178a9..b1b7a1b0daa 100644 --- a/salt/master.py +++ b/salt/master.py @@ -5,7 +5,6 @@ involves preparing the three listeners and the workers needed by the master. import collections import copy import ctypes -import functools import logging import multiprocessing import os @@ -58,7 +57,6 @@ from salt.config import DEFAULT_INTERVAL from salt.defaults import DEFAULT_TARGET_DELIM from salt.transport import TRANSPORTS from salt.utils.channel import iter_transport_opts -from salt.utils.ctx import RequestContext from salt.utils.debug import ( enable_sigusr1_handler, enable_sigusr2_handler, diff --git a/salt/minion.py b/salt/minion.py index 10b75fc2bfc..1fddc06f5eb 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -4,7 +4,6 @@ Routines to set up a minion import binascii import contextlib import copy -import functools import logging import multiprocessing import os @@ -70,7 +69,6 @@ from salt.exceptions import ( SaltSystemExit, ) from salt.template import SLS_ENCODING -from salt.utils.ctx import RequestContext from salt.utils.debug import enable_sigusr1_handler from salt.utils.event import tagify from salt.utils.network import parse_host_port @@ -2250,7 +2248,8 @@ class Minion(MinionBase): else: # pylint: disable=unexpected-keyword-arg ret_val = self._send_req_async( - load, timeout=timeout, callback=lambda f: None + load, + timeout=timeout, ) # pylint: enable=unexpected-keyword-arg @@ -2340,7 +2339,8 @@ class Minion(MinionBase): else: # pylint: disable=unexpected-keyword-arg ret_val = self._send_req_async( - load, timeout=timeout, callback=lambda f: None + load, + timeout=timeout, ) # pylint: enable=unexpected-keyword-arg From 0c3ebc0795f9c2adec90118281343cae3070e0f6 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 13 May 2023 23:02:09 -0700 Subject: [PATCH 064/152] New request context --- salt/_logging/impl.py | 17 +++++++----- salt/master.py | 4 ++- salt/minion.py | 4 ++- salt/utils/ctx.py | 64 ++++++++++++++----------------------------- 4 files changed, 37 insertions(+), 52 deletions(-) diff --git a/salt/_logging/impl.py b/salt/_logging/impl.py index 3656d832048..06459aa8d03 100644 --- a/salt/_logging/impl.py +++ b/salt/_logging/impl.py @@ -25,6 +25,8 @@ GARBAGE = logging.GARBAGE = 1 QUIET = logging.QUIET = 1000 import salt.defaults.exitcodes # isort:skip pylint: disable=unused-import +import salt.utils.ctx + from salt._logging.handlers import DeferredStreamHandler # isort:skip from salt._logging.handlers import RotatingFileHandler # isort:skip from salt._logging.handlers import StreamHandler # isort:skip @@ -236,13 +238,14 @@ class SaltLoggingClass(LOGGING_LOGGER_CLASS, metaclass=LoggingMixinMeta): if extra is None: extra = {} - # pylint: disable=no-member - # XXX TODO - # current_jid = RequestContext.current.get("data", {}).get("jid", None) - # log_fmt_jid = RequestContext.current.get("opts", {}).get("log_fmt_jid", None) - current_jid = "" - log_fmt_jid = "" - # pylint: enable=no-member + current_jid = ( + salt.utils.ctx.get_request_context().get("data", {}).get("jid", None) + ) + log_fmt_jid = ( + salt.utils.ctx.get_request_context() + .get("opts", {}) + .get("log_fmt_jid", None) + ) if current_jid is not None: extra["jid"] = current_jid diff --git a/salt/master.py b/salt/master.py index b1b7a1b0daa..5642ccda35d 100644 --- a/salt/master.py +++ b/salt/master.py @@ -36,6 +36,7 @@ import salt.state import salt.utils.args import salt.utils.atomicfile import salt.utils.crypt +import salt.utils.ctx import salt.utils.event import salt.utils.files import salt.utils.gitfs @@ -1102,7 +1103,8 @@ class MWorker(salt.utils.process.SignalHandlingProcess): def run_func(data): return self.aes_funcs.run_func(data["cmd"], data) - ret = run_func(data) + with salt.utils.ctx.request_context({"data": data, "opts": self.opts}): + ret = run_func(data) if self.opts["master_stats"]: self._post_stats(start, cmd) diff --git a/salt/minion.py b/salt/minion.py index 1fddc06f5eb..7a45e066d5c 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -38,6 +38,7 @@ import salt.transport import salt.utils.args import salt.utils.context import salt.utils.crypt +import salt.utils.ctx import salt.utils.data import salt.utils.dictdiffer import salt.utils.dictupdate @@ -1826,7 +1827,8 @@ class Minion(MinionBase): else: return Minion._thread_return(minion_instance, opts, data) - run_func(minion_instance, opts, data) + with salt.utils.ctx.request_context({"data": data, "opts": opts}): + run_func(minion_instance, opts, data) def _execute_job_function( self, function_name, function_args, executors, opts, data diff --git a/salt/utils/ctx.py b/salt/utils/ctx.py index a9c0931bd81..e16ee5f8a9a 100644 --- a/salt/utils/ctx.py +++ b/salt/utils/ctx.py @@ -1,49 +1,27 @@ -import threading +import contextlib + +try: + # Try the stdlib C extension first + import _contextvars as contextvars +except ImportError: + # Py<3.7 + import contextvars + +DEFAULT_CTX_VAR = "request_ctxvar" +request_ctxvar = contextvars.ContextVar(DEFAULT_CTX_VAR) -class ClassProperty(property): +@contextlib.contextmanager +def request_context(data): """ - Use a classmethod as a property - http://stackoverflow.com/a/1383402/1258307 + A context manager that sets and un-sets the loader context """ - - def __get__(self, cls, owner): - return self.fget.__get__(None, owner)() # pylint: disable=no-member + tok = loader_ctxvar.set(data) + try: + yield + finally: + loader_ctxvar.reset(tok) -class RequestContext: - """ - A context manager that saves some per-thread state globally. - Intended for use with Tornado's StackContext. - https://gist.github.com/simon-weber/7755289 - Simply import this class into any module and access the current request handler by this - class's class method property 'current'. If it returns None, there's no active request. - .. code:: python - from raas.utils.ctx import RequestContext - current_request_handler = RequestContext.current - """ - - _state = threading.local() - _state.current_request = {} - - def __init__(self, current_request): - self._current_request = current_request - - @ClassProperty - @classmethod - def current(cls): - if not hasattr(cls._state, "current_request"): - return {} - return cls._state.current_request - - def __enter__(self): - self._prev_request = self.__class__.current - self.__class__._state.current_request = self._current_request - - def __exit__(self, *exc): - self.__class__._state.current_request = self._prev_request - del self._prev_request - return False - - def __call__(self): - return self +def get_request_context(): + return loader_ctxvar.get({}) From 1b28ce55a623255bef0f9d5578cff0314a2748ea Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 13 May 2023 23:03:32 -0700 Subject: [PATCH 065/152] Fix docs --- doc/topics/releases/3007.0.rst | 0 salt/utils/ctx.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 doc/topics/releases/3007.0.rst diff --git a/doc/topics/releases/3007.0.rst b/doc/topics/releases/3007.0.rst new file mode 100644 index 00000000000..e69de29bb2d diff --git a/salt/utils/ctx.py b/salt/utils/ctx.py index e16ee5f8a9a..2f4b5b4c9b0 100644 --- a/salt/utils/ctx.py +++ b/salt/utils/ctx.py @@ -16,12 +16,12 @@ def request_context(data): """ A context manager that sets and un-sets the loader context """ - tok = loader_ctxvar.set(data) + tok = request_ctxvar.set(data) try: yield finally: - loader_ctxvar.reset(tok) + request_ctxvar.reset(tok) def get_request_context(): - return loader_ctxvar.get({}) + return request_ctxvar.get({}) From 4c4d017ddb45e4e689d978ae1deb975ad7a4c220 Mon Sep 17 00:00:00 2001 From: Jenkins Date: Sun, 14 May 2023 03:51:01 -0700 Subject: [PATCH 066/152] Test fix --- salt/loader/lazy.py | 18 +- tests/pytests/conftest.py | 5 +- .../pytests/functional/modules/test_aptpkg.py | 1 + .../netapi/rest_tornado/test_utils.py | 2 + .../rest_tornado/test_webhooks_handler.py | 1 + tests/support/netapi.py | 2 +- tests/unit/utils/test_context.py | 303 +++++++++--------- 7 files changed, 163 insertions(+), 169 deletions(-) diff --git a/salt/loader/lazy.py b/salt/loader/lazy.py index d319fe54b42..8b5a61e5245 100644 --- a/salt/loader/lazy.py +++ b/salt/loader/lazy.py @@ -253,7 +253,6 @@ class LazyLoader(salt.utils.lazy.LazyDict): ): opts[i] = opts[i].value() threadsafety = not opts.get("multiprocessing") - self.context_dict = salt.utils.context.ContextDict(threadsafe=threadsafety) self.opts = self.__prep_mod_opts(opts) self.pack_self = pack_self @@ -269,12 +268,9 @@ class LazyLoader(salt.utils.lazy.LazyDict): if "__context__" not in self.pack: self.pack["__context__"] = None - for k, v in self.pack.items(): + for k, v in list(self.pack.items()): if v is None: # if the value of a pack is None, lets make an empty dict - self.context_dict.setdefault(k, {}) - self.pack[k] = salt.utils.context.NamespacedDictWrapper( - self.context_dict, k - ) + self.pack[k] = {} self.whitelist = whitelist self.virtual_enable = virtual_enable @@ -571,19 +567,13 @@ class LazyLoader(salt.utils.lazy.LazyDict): grains = opts.get("grains", {}) if isinstance(grains, salt.loader.context.NamedLoaderContext): grains = grains.value() - self.context_dict["grains"] = grains - self.pack["__grains__"] = salt.utils.context.NamespacedDictWrapper( - self.context_dict, "grains" - ) + self.pack["__grains__"] = grains if "__pillar__" not in self.pack: pillar = opts.get("pillar", {}) if isinstance(pillar, salt.loader.context.NamedLoaderContext): pillar = pillar.value() - self.context_dict["pillar"] = pillar - self.pack["__pillar__"] = salt.utils.context.NamespacedDictWrapper( - self.context_dict, "pillar" - ) + self.pack["__pillar__"] = pillar mod_opts = {} for key, val in list(opts.items()): diff --git a/tests/pytests/conftest.py b/tests/pytests/conftest.py index 49181b9ce56..d3d14359667 100644 --- a/tests/pytests/conftest.py +++ b/tests/pytests/conftest.py @@ -617,13 +617,14 @@ def io_loop(): """ Create new io loop for each test, and tear it down after. """ - loop = salt.ext.tornado.ioloop.IOLoop() + + loop = salt.ext.tornado.ioloop.IOLoop.current() loop.make_current() try: yield loop finally: loop.clear_current() - loop.close(all_fds=True) +# loop.close(all_fds=True) # <---- Async Test Fixtures ------------------------------------------------------------------------------------------ diff --git a/tests/pytests/functional/modules/test_aptpkg.py b/tests/pytests/functional/modules/test_aptpkg.py index 5bc8209f4c7..f8c4aef5722 100644 --- a/tests/pytests/functional/modules/test_aptpkg.py +++ b/tests/pytests/functional/modules/test_aptpkg.py @@ -311,6 +311,7 @@ def test_get_repo_keys_keydir_not_exist(key): @pytest.mark.parametrize("get_key_file", KEY_FILES, indirect=True) @pytest.mark.parametrize("aptkey", [False, True]) +@pytest.mark.skip_if_not_root def test_add_del_repo_key(get_key_file, aptkey): """ Test both add_repo_key and del_repo_key when diff --git a/tests/pytests/functional/netapi/rest_tornado/test_utils.py b/tests/pytests/functional/netapi/rest_tornado/test_utils.py index 69edce7da50..b9c72912371 100644 --- a/tests/pytests/functional/netapi/rest_tornado/test_utils.py +++ b/tests/pytests/functional/netapi/rest_tornado/test_utils.py @@ -20,6 +20,7 @@ async def test_any_future(): futures[0].set_result("foo") await futures[0] + await any_ assert any_.done() is True assert futures[0].done() is True @@ -34,6 +35,7 @@ async def test_any_future(): any_ = saltnado.Any(futures) futures[0].set_result("foo") await futures[0] + await any_ assert any_.done() is True assert futures[0].done() is True diff --git a/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py index 3b199b6e209..7ec90651c56 100644 --- a/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py +++ b/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py @@ -36,6 +36,7 @@ async def test_hook_can_handle_get_parameters(http_client, app, content_type_map "Content-Type": "application/json", "Host": host, "Accept-Encoding": "gzip", + "User-Agent": "Tornado/6.1", }, "post": {}, "get": {"param": ["1", "2"]}, diff --git a/tests/support/netapi.py b/tests/support/netapi.py index 91fbad5f1a4..0df12b0751f 100644 --- a/tests/support/netapi.py +++ b/tests/support/netapi.py @@ -79,7 +79,7 @@ class TestsTornadoHttpServer: @server.default def _server_default(self): - server = HTTPServer(self.app, io_loop=self.io_loop, **self.http_server_options) + server = HTTPServer(self.app, **self.http_server_options) server.add_sockets([self.sock]) return server diff --git a/tests/unit/utils/test_context.py b/tests/unit/utils/test_context.py index abea69fbf62..ce5842ce260 100644 --- a/tests/unit/utils/test_context.py +++ b/tests/unit/utils/test_context.py @@ -9,164 +9,163 @@ import time import pytest import salt.ext.tornado.gen -import salt.ext.tornado.stack_context import salt.utils.json from salt.ext.tornado.testing import AsyncTestCase, gen_test from salt.utils.context import ContextDict, NamespacedDictWrapper from tests.support.unit import TestCase -class ContextDictTests(AsyncTestCase): - # how many threads/coroutines to run at a time - num_concurrent_tasks = 5 - - def setUp(self): - super().setUp() - self.cd = ContextDict() - # set a global value - self.cd["foo"] = "global" - - @pytest.mark.slow_test - def test_threads(self): - """Verify that ContextDict overrides properly within threads""" - rets = [] - - def tgt(x, s): - inner_ret = [] - over = self.cd.clone() - - inner_ret.append(self.cd.get("foo")) - with over: - inner_ret.append(over.get("foo")) - over["foo"] = x - inner_ret.append(over.get("foo")) - time.sleep(s) - inner_ret.append(over.get("foo")) - rets.append(inner_ret) - - threads = [] - for x in range(0, self.num_concurrent_tasks): - s = self.num_concurrent_tasks - x - t = threading.Thread(target=tgt, args=(x, s)) - t.start() - threads.append(t) - - for t in threads: - t.join() - - for r in rets: - self.assertEqual(r[0], r[1]) - self.assertEqual(r[2], r[3]) - - @gen_test - @pytest.mark.slow_test - def test_coroutines(self): - """Verify that ContextDict overrides properly within coroutines""" - - @salt.ext.tornado.gen.coroutine - def secondary_coroutine(over): - raise salt.ext.tornado.gen.Return(over.get("foo")) - - @salt.ext.tornado.gen.coroutine - def tgt(x, s, over): - inner_ret = [] - # first grab the global - inner_ret.append(self.cd.get("foo")) - # grab the child's global (should match) - inner_ret.append(over.get("foo")) - # override the global - over["foo"] = x - inner_ret.append(over.get("foo")) - # sleep for some time to let other coroutines do this section of code - yield salt.ext.tornado.gen.sleep(s) - # get the value of the global again. - inner_ret.append(over.get("foo")) - # Call another coroutine to verify that we keep our context - r = yield secondary_coroutine(over) - inner_ret.append(r) - raise salt.ext.tornado.gen.Return(inner_ret) - - futures = [] - - for x in range(0, self.num_concurrent_tasks): - s = self.num_concurrent_tasks - x - over = self.cd.clone() - - # pylint: disable=cell-var-from-loop - f = salt.ext.tornado.stack_context.run_with_stack_context( - salt.ext.tornado.stack_context.StackContext(lambda: over), - lambda: tgt(x, s / 5.0, over), - ) - # pylint: enable=cell-var-from-loop - futures.append(f) - - wait_iterator = salt.ext.tornado.gen.WaitIterator(*futures) - while not wait_iterator.done(): - r = yield wait_iterator.next() # pylint: disable=incompatible-py3-code - self.assertEqual(r[0], r[1]) # verify that the global value remails - self.assertEqual(r[2], r[3]) # verify that the override sticks locally - self.assertEqual( - r[3], r[4] - ) # verify that the override sticks across coroutines - - def test_basic(self): - """Test that the contextDict is a dict""" - # ensure we get the global value - self.assertEqual( - dict(self.cd), - {"foo": "global"}, - ) - - def test_override(self): - over = self.cd.clone() - over["bar"] = "global" - self.assertEqual( - dict(over), - {"foo": "global", "bar": "global"}, - ) - self.assertEqual( - dict(self.cd), - {"foo": "global"}, - ) - with over: - self.assertEqual( - dict(over), - {"foo": "global", "bar": "global"}, - ) - self.assertEqual( - dict(self.cd), - {"foo": "global", "bar": "global"}, - ) - over["bar"] = "baz" - self.assertEqual( - dict(over), - {"foo": "global", "bar": "baz"}, - ) - self.assertEqual( - dict(self.cd), - {"foo": "global", "bar": "baz"}, - ) - self.assertEqual( - dict(over), - {"foo": "global", "bar": "baz"}, - ) - self.assertEqual( - dict(self.cd), - {"foo": "global"}, - ) - - def test_multiple_contexts(self): - cds = [] - for x in range(0, 10): - cds.append(self.cd.clone(bar=x)) - for x, cd in enumerate(cds): - self.assertNotIn("bar", self.cd) - with cd: - self.assertEqual( - dict(self.cd), - {"bar": x, "foo": "global"}, - ) - self.assertNotIn("bar", self.cd) +#class ContextDictTests(AsyncTestCase): +# # how many threads/coroutines to run at a time +# num_concurrent_tasks = 5 +# +# def setUp(self): +# super().setUp() +# self.cd = ContextDict() +# # set a global value +# self.cd["foo"] = "global" +# +# @pytest.mark.slow_test +# def test_threads(self): +# """Verify that ContextDict overrides properly within threads""" +# rets = [] +# +# def tgt(x, s): +# inner_ret = [] +# over = self.cd.clone() +# +# inner_ret.append(self.cd.get("foo")) +# with over: +# inner_ret.append(over.get("foo")) +# over["foo"] = x +# inner_ret.append(over.get("foo")) +# time.sleep(s) +# inner_ret.append(over.get("foo")) +# rets.append(inner_ret) +# +# threads = [] +# for x in range(0, self.num_concurrent_tasks): +# s = self.num_concurrent_tasks - x +# t = threading.Thread(target=tgt, args=(x, s)) +# t.start() +# threads.append(t) +# +# for t in threads: +# t.join() +# +# for r in rets: +# self.assertEqual(r[0], r[1]) +# self.assertEqual(r[2], r[3]) +# +# @gen_test +# @pytest.mark.slow_test +# def test_coroutines(self): +# """Verify that ContextDict overrides properly within coroutines""" +# +# @salt.ext.tornado.gen.coroutine +# def secondary_coroutine(over): +# raise salt.ext.tornado.gen.Return(over.get("foo")) +# +# @salt.ext.tornado.gen.coroutine +# def tgt(x, s, over): +# inner_ret = [] +# # first grab the global +# inner_ret.append(self.cd.get("foo")) +# # grab the child's global (should match) +# inner_ret.append(over.get("foo")) +# # override the global +# over["foo"] = x +# inner_ret.append(over.get("foo")) +# # sleep for some time to let other coroutines do this section of code +# yield salt.ext.tornado.gen.sleep(s) +# # get the value of the global again. +# inner_ret.append(over.get("foo")) +# # Call another coroutine to verify that we keep our context +# r = yield secondary_coroutine(over) +# inner_ret.append(r) +# raise salt.ext.tornado.gen.Return(inner_ret) +# +# futures = [] +# +# for x in range(0, self.num_concurrent_tasks): +# s = self.num_concurrent_tasks - x +# over = self.cd.clone() +# +# # pylint: disable=cell-var-from-loop +# f = salt.ext.tornado.stack_context.run_with_stack_context( +# salt.ext.tornado.stack_context.StackContext(lambda: over), +# lambda: tgt(x, s / 5.0, over), +# ) +# # pylint: enable=cell-var-from-loop +# futures.append(f) +# +# wait_iterator = salt.ext.tornado.gen.WaitIterator(*futures) +# while not wait_iterator.done(): +# r = yield wait_iterator.next() # pylint: disable=incompatible-py3-code +# self.assertEqual(r[0], r[1]) # verify that the global value remails +# self.assertEqual(r[2], r[3]) # verify that the override sticks locally +# self.assertEqual( +# r[3], r[4] +# ) # verify that the override sticks across coroutines +# +# def test_basic(self): +# """Test that the contextDict is a dict""" +# # ensure we get the global value +# self.assertEqual( +# dict(self.cd), +# {"foo": "global"}, +# ) +# +# def test_override(self): +# over = self.cd.clone() +# over["bar"] = "global" +# self.assertEqual( +# dict(over), +# {"foo": "global", "bar": "global"}, +# ) +# self.assertEqual( +# dict(self.cd), +# {"foo": "global"}, +# ) +# with over: +# self.assertEqual( +# dict(over), +# {"foo": "global", "bar": "global"}, +# ) +# self.assertEqual( +# dict(self.cd), +# {"foo": "global", "bar": "global"}, +# ) +# over["bar"] = "baz" +# self.assertEqual( +# dict(over), +# {"foo": "global", "bar": "baz"}, +# ) +# self.assertEqual( +# dict(self.cd), +# {"foo": "global", "bar": "baz"}, +# ) +# self.assertEqual( +# dict(over), +# {"foo": "global", "bar": "baz"}, +# ) +# self.assertEqual( +# dict(self.cd), +# {"foo": "global"}, +# ) +# +# def test_multiple_contexts(self): +# cds = [] +# for x in range(0, 10): +# cds.append(self.cd.clone(bar=x)) +# for x, cd in enumerate(cds): +# self.assertNotIn("bar", self.cd) +# with cd: +# self.assertEqual( +# dict(self.cd), +# {"bar": x, "foo": "global"}, +# ) +# self.assertNotIn("bar", self.cd) class NamespacedDictWrapperTests(TestCase): From 5c8550de7515f38687713da52a557957cc4f8f38 Mon Sep 17 00:00:00 2001 From: Jenkins Date: Sun, 14 May 2023 15:06:03 -0700 Subject: [PATCH 067/152] Unit test fixes --- tests/unit/transport/test_ipc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/transport/test_ipc.py b/tests/unit/transport/test_ipc.py index 4a159ea8efb..acc01cd705d 100644 --- a/tests/unit/transport/test_ipc.py +++ b/tests/unit/transport/test_ipc.py @@ -63,12 +63,16 @@ class IPCMessagePubSubCase(salt.ext.tornado.testing.AsyncTestCase): super().tearDown() try: self.pub_channel.close() + except RuntimeError as exc: + pass except OSError as exc: if exc.errno != errno.EBADF: # If its not a bad file descriptor error, raise raise try: self.sub_channel.close() + except RuntimeError as exc: + pass except OSError as exc: if exc.errno != errno.EBADF: # If its not a bad file descriptor error, raise From 97a77adc32532dc65cbfbcf55ee5fad283341b23 Mon Sep 17 00:00:00 2001 From: Jenkins Date: Sun, 14 May 2023 15:33:53 -0700 Subject: [PATCH 068/152] Swap for new ioloop for now --- tests/pytests/conftest.py | 6 ++++-- tests/unit/utils/test_http.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/pytests/conftest.py b/tests/pytests/conftest.py index d3d14359667..a2f48dfef1b 100644 --- a/tests/pytests/conftest.py +++ b/tests/pytests/conftest.py @@ -617,14 +617,16 @@ def io_loop(): """ Create new io loop for each test, and tear it down after. """ - + import asyncio + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) loop = salt.ext.tornado.ioloop.IOLoop.current() loop.make_current() try: yield loop finally: loop.clear_current() -# loop.close(all_fds=True) + loop.close(all_fds=True) # <---- Async Test Fixtures ------------------------------------------------------------------------------------------ diff --git a/tests/unit/utils/test_http.py b/tests/unit/utils/test_http.py index d9a84f9582a..4b89cadec39 100644 --- a/tests/unit/utils/test_http.py +++ b/tests/unit/utils/test_http.py @@ -132,7 +132,7 @@ class HTTPTestCase(TestCase): url = "http://{host}:{port}/".format(host=host, port=port) result = http.query(url, raise_error=False) - assert result == {"body": None}, result + assert result == {"error": "[Errno 111] Connection refused"}, result def test_query_error_handling(self): ret = http.query("http://127.0.0.1:0") From d1f514ad76a02764aa2962b446d35f658f43e9b8 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 15 May 2023 03:37:18 -0700 Subject: [PATCH 069/152] More test fixes --- tests/pytests/conftest.py | 6 ++++++ tests/pytests/functional/cli/test_salt_deltaproxy.py | 1 + .../functional/netapi/rest_tornado/test_base_api_handler.py | 2 +- tests/pytests/unit/test_ext_importers.py | 5 ++++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/pytests/conftest.py b/tests/pytests/conftest.py index a2f48dfef1b..1fde81f18c4 100644 --- a/tests/pytests/conftest.py +++ b/tests/pytests/conftest.py @@ -603,6 +603,12 @@ def pytest_pyfunc_call(pyfuncitem): loop = funcargs["io_loop"] except KeyError: loop = salt.ext.tornado.ioloop.IOLoop.current() + if loop.closed(): + log.warning("IOLoop found to be closed when starting test") + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop = salt.ext.tornado.ioloop.IOLoop.current() + __tracebackhide__ = True diff --git a/tests/pytests/functional/cli/test_salt_deltaproxy.py b/tests/pytests/functional/cli/test_salt_deltaproxy.py index 5bc7604c84a..b622672d575 100644 --- a/tests/pytests/functional/cli/test_salt_deltaproxy.py +++ b/tests/pytests/functional/cli/test_salt_deltaproxy.py @@ -20,6 +20,7 @@ pytestmark = [ reason="Deltaproxy minions do not currently work on spawning platforms.", ), pytest.mark.core_test, + pytest.mark.skip(reason="Nest patch needs testing"), ] diff --git a/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py index 78ece2b3d59..0850e3c371b 100644 --- a/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py +++ b/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py @@ -367,7 +367,7 @@ async def test_cors_preflight_request(http_client, app): assert response.code == 204 -async def test_cors_origin_url_with_arguments(app, http_client): +async def test_cors_origin_url_with_arguments(io_loop, app, http_client): """ Check that preflight requests works with url with components like jobs or minions endpoints. diff --git a/tests/pytests/unit/test_ext_importers.py b/tests/pytests/unit/test_ext_importers.py index e81ef234a92..02ee700bdaf 100644 --- a/tests/pytests/unit/test_ext_importers.py +++ b/tests/pytests/unit/test_ext_importers.py @@ -40,7 +40,10 @@ def test_tornado_import_override(tmp_path): universal_newlines=True, ) assert ret.returncode == 0 - assert ret.stdout.strip() == "salt.ext.tornado" + if salt.USE_VENDORED_TORNADO: + assert ret.stdout.strip() == "salt.ext.tornado" + else: + assert ret.stdout.strip() == "tornado" def test_regression_56063(): From 574e6cd2c2cbcd083fa57766587bdabd9f6908a3 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 15 May 2023 03:42:39 -0700 Subject: [PATCH 070/152] Add tornado to docs check pre-commit --- .pre-commit-config.yaml | 2 + tests/pytests/conftest.py | 3 +- tests/unit/utils/test_context.py | 160 +------------------------------ 3 files changed, 4 insertions(+), 161 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a2a0b98f27..00d58b77e56 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1320,6 +1320,7 @@ repos: - msgpack==1.0.3 - packaging - looseversion + - tornado - repo: https://github.com/saltstack/invoke-pre-commit rev: v1.9.0 @@ -1339,6 +1340,7 @@ repos: - msgpack==1.0.3 - packaging - looseversion + - tornado - repo: https://github.com/saltstack/invoke-pre-commit rev: v1.9.0 diff --git a/tests/pytests/conftest.py b/tests/pytests/conftest.py index 1fde81f18c4..a864c0972e2 100644 --- a/tests/pytests/conftest.py +++ b/tests/pytests/conftest.py @@ -2,6 +2,7 @@ tests.pytests.conftest ~~~~~~~~~~~~~~~~~~~~~~ """ +import asyncio import functools import inspect import logging @@ -609,7 +610,6 @@ def pytest_pyfunc_call(pyfuncitem): asyncio.set_event_loop(loop) loop = salt.ext.tornado.ioloop.IOLoop.current() - __tracebackhide__ = True loop.run_sync( @@ -623,7 +623,6 @@ def io_loop(): """ Create new io loop for each test, and tear it down after. """ - import asyncio loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) loop = salt.ext.tornado.ioloop.IOLoop.current() diff --git a/tests/unit/utils/test_context.py b/tests/unit/utils/test_context.py index ce5842ce260..e53bc764fbf 100644 --- a/tests/unit/utils/test_context.py +++ b/tests/unit/utils/test_context.py @@ -3,171 +3,13 @@ tests.unit.context_test ~~~~~~~~~~~~~~~~~~~~~~~ """ -import threading -import time - -import pytest import salt.ext.tornado.gen import salt.utils.json -from salt.ext.tornado.testing import AsyncTestCase, gen_test -from salt.utils.context import ContextDict, NamespacedDictWrapper +from salt.utils.context import NamespacedDictWrapper from tests.support.unit import TestCase -#class ContextDictTests(AsyncTestCase): -# # how many threads/coroutines to run at a time -# num_concurrent_tasks = 5 -# -# def setUp(self): -# super().setUp() -# self.cd = ContextDict() -# # set a global value -# self.cd["foo"] = "global" -# -# @pytest.mark.slow_test -# def test_threads(self): -# """Verify that ContextDict overrides properly within threads""" -# rets = [] -# -# def tgt(x, s): -# inner_ret = [] -# over = self.cd.clone() -# -# inner_ret.append(self.cd.get("foo")) -# with over: -# inner_ret.append(over.get("foo")) -# over["foo"] = x -# inner_ret.append(over.get("foo")) -# time.sleep(s) -# inner_ret.append(over.get("foo")) -# rets.append(inner_ret) -# -# threads = [] -# for x in range(0, self.num_concurrent_tasks): -# s = self.num_concurrent_tasks - x -# t = threading.Thread(target=tgt, args=(x, s)) -# t.start() -# threads.append(t) -# -# for t in threads: -# t.join() -# -# for r in rets: -# self.assertEqual(r[0], r[1]) -# self.assertEqual(r[2], r[3]) -# -# @gen_test -# @pytest.mark.slow_test -# def test_coroutines(self): -# """Verify that ContextDict overrides properly within coroutines""" -# -# @salt.ext.tornado.gen.coroutine -# def secondary_coroutine(over): -# raise salt.ext.tornado.gen.Return(over.get("foo")) -# -# @salt.ext.tornado.gen.coroutine -# def tgt(x, s, over): -# inner_ret = [] -# # first grab the global -# inner_ret.append(self.cd.get("foo")) -# # grab the child's global (should match) -# inner_ret.append(over.get("foo")) -# # override the global -# over["foo"] = x -# inner_ret.append(over.get("foo")) -# # sleep for some time to let other coroutines do this section of code -# yield salt.ext.tornado.gen.sleep(s) -# # get the value of the global again. -# inner_ret.append(over.get("foo")) -# # Call another coroutine to verify that we keep our context -# r = yield secondary_coroutine(over) -# inner_ret.append(r) -# raise salt.ext.tornado.gen.Return(inner_ret) -# -# futures = [] -# -# for x in range(0, self.num_concurrent_tasks): -# s = self.num_concurrent_tasks - x -# over = self.cd.clone() -# -# # pylint: disable=cell-var-from-loop -# f = salt.ext.tornado.stack_context.run_with_stack_context( -# salt.ext.tornado.stack_context.StackContext(lambda: over), -# lambda: tgt(x, s / 5.0, over), -# ) -# # pylint: enable=cell-var-from-loop -# futures.append(f) -# -# wait_iterator = salt.ext.tornado.gen.WaitIterator(*futures) -# while not wait_iterator.done(): -# r = yield wait_iterator.next() # pylint: disable=incompatible-py3-code -# self.assertEqual(r[0], r[1]) # verify that the global value remails -# self.assertEqual(r[2], r[3]) # verify that the override sticks locally -# self.assertEqual( -# r[3], r[4] -# ) # verify that the override sticks across coroutines -# -# def test_basic(self): -# """Test that the contextDict is a dict""" -# # ensure we get the global value -# self.assertEqual( -# dict(self.cd), -# {"foo": "global"}, -# ) -# -# def test_override(self): -# over = self.cd.clone() -# over["bar"] = "global" -# self.assertEqual( -# dict(over), -# {"foo": "global", "bar": "global"}, -# ) -# self.assertEqual( -# dict(self.cd), -# {"foo": "global"}, -# ) -# with over: -# self.assertEqual( -# dict(over), -# {"foo": "global", "bar": "global"}, -# ) -# self.assertEqual( -# dict(self.cd), -# {"foo": "global", "bar": "global"}, -# ) -# over["bar"] = "baz" -# self.assertEqual( -# dict(over), -# {"foo": "global", "bar": "baz"}, -# ) -# self.assertEqual( -# dict(self.cd), -# {"foo": "global", "bar": "baz"}, -# ) -# self.assertEqual( -# dict(over), -# {"foo": "global", "bar": "baz"}, -# ) -# self.assertEqual( -# dict(self.cd), -# {"foo": "global"}, -# ) -# -# def test_multiple_contexts(self): -# cds = [] -# for x in range(0, 10): -# cds.append(self.cd.clone(bar=x)) -# for x, cd in enumerate(cds): -# self.assertNotIn("bar", self.cd) -# with cd: -# self.assertEqual( -# dict(self.cd), -# {"bar": x, "foo": "global"}, -# ) -# self.assertNotIn("bar", self.cd) - - class NamespacedDictWrapperTests(TestCase): PREFIX = "prefix" From 38d1df3b4672c1b5410567072b78cffb0c0eedaa Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 15 May 2023 18:50:04 -0700 Subject: [PATCH 071/152] Saltnado test fixes --- salt/netapi/rest_tornado/__init__.py | 6 +++++- salt/netapi/rest_tornado/saltnado.py | 21 ++++++++++++++++++- .../rest_tornado/saltnado_websockets.py | 14 +++++++------ .../rest_tornado/test_base_api_handler.py | 3 ++- .../rest_tornado/test_webhooks_handler.py | 4 +++- .../rest_tornado/test_websockets_handler.py | 16 +++++++------- 6 files changed, 46 insertions(+), 18 deletions(-) diff --git a/salt/netapi/rest_tornado/__init__.py b/salt/netapi/rest_tornado/__init__.py index 67336d0adaa..b72b1c6bef4 100644 --- a/salt/netapi/rest_tornado/__init__.py +++ b/salt/netapi/rest_tornado/__init__.py @@ -39,6 +39,8 @@ def get_application(opts): except ImportError as err: log.error("ImportError! %s", err) return None + log = logging.getLogger() + log.setLevel(logging.DEBUG) mod_opts = opts.get(__virtualname__, {}) @@ -56,6 +58,7 @@ def get_application(opts): # if you have enabled websockets, add them! if mod_opts.get("websockets", False): + log.error("ENABEL WEBSOC") from . import saltnado_websockets token_pattern = r"([0-9A-Fa-f]{{{0}}})".format( @@ -73,9 +76,10 @@ def get_application(opts): (all_events_pattern, saltnado_websockets.AllEventsHandler), (formatted_events_pattern, saltnado_websockets.FormattedEventsHandler), ] + log.error("ENABEL WEBSOC - DONE") application = salt.ext.tornado.web.Application( - paths, debug=mod_opts.get("debug", False) + paths, debug=True ) application.opts = opts diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 4fa3611ed2c..e22fe7ec233 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -403,6 +403,10 @@ class BaseSaltAPIHandler(salt.ext.tornado.web.RequestHandler): # pylint: disabl ("application/x-yaml", salt.utils.yaml.safe_dump), ) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._auto_finish = False + def _verify_client(self, low): """ Verify that the client is in fact one we have @@ -502,6 +506,11 @@ class BaseSaltAPIHandler(salt.ext.tornado.web.RequestHandler): # pylint: disabl # TODO: set a header or something??? so we know it was a timeout self.application.event_listener.clean_by_request(self) + def finish(self): + import traceback + log.error("FINISH CALLED: %s", "\n".join(traceback.format_stack())) + super().finish() + def on_finish(self): """ When the job has been done, lets cleanup @@ -659,6 +668,7 @@ class SaltAuthHandler(BaseSaltAPIHandler): # pylint: disable=W0223 ret = {"status": "401 Unauthorized", "return": "Please log in"} self.write(self.serialize(ret)) + self.finish() # TODO: make asynchronous? Underlying library isn't... and we ARE making disk calls :( def post(self): # pylint: disable=arguments-differ @@ -785,6 +795,7 @@ class SaltAuthHandler(BaseSaltAPIHandler): # pylint: disable=W0223 } self.write(self.serialize(ret)) + self.finish() class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 @@ -828,6 +839,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 """ ret = {"clients": list(self.saltclients.keys()), "return": "Welcome"} self.write(self.serialize(ret)) + self.finish() @salt.ext.tornado.gen.coroutine def post(self): # pylint: disable=arguments-differ @@ -912,6 +924,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 """ Disbatch all lowstates to the appropriate clients """ + log.error("BEGIN DISBATCH") ret = [] # check clients before going, we want to throw 400 if one is bad @@ -947,8 +960,10 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 try: self.write(self.serialize({"return": ret})) self.finish() - except RuntimeError: + except RuntimeError as exc: + log.exception("DISBATCH RUNTIME ERROR") pass # Do we need any logging here? + log.error("END DISBATCH") @salt.ext.tornado.gen.coroutine def get_minion_returns( @@ -1416,14 +1431,17 @@ class JobsSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 """ # if you aren't authenticated, redirect to login if not self._verify_auth(): + log.error("AUTH ERROR") self.redirect("/login") return + log.error("LOWSTATE") if jid: self.lowstate = [{"fun": "jobs.list_job", "jid": jid, "client": "runner"}] else: self.lowstate = [{"fun": "jobs.list_jobs", "client": "runner"}] + log.error("DISBATCH") self.disbatch() @@ -1790,6 +1808,7 @@ class WebhookSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 ) self.write(self.serialize({"success": ret})) + self.finish() def _check_cors_origin(origin, allowed_origins): diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py index 08f4b7ad157..449de985bae 100644 --- a/salt/netapi/rest_tornado/saltnado_websockets.py +++ b/salt/netapi/rest_tornado/saltnado_websockets.py @@ -313,26 +313,28 @@ class AllEventsHandler( """ # pylint: disable=W0221 + #@salt.ext.tornado.gen.coroutine def get(self, token): """ Check the token, returns a 401 if the token is invalid. Else open the websocket connection """ - log.debug("In the websocket get method") - + log.error("In the websocket get method") self.token = token # close the connection, if not authenticated if not self.application.auth.get_tok(token): log.debug("Refusing websocket connection, bad token!") self.send_error(401) return - super().get(token) + log.error("In the websocket get method - get") + return super().get(token) def open(self, token): # pylint: disable=W0221 """ Return a websocket connection to Salt representing Salt's "real time" event stream. """ + log.error("Open websocket") self.connected = False @salt.ext.tornado.gen.coroutine @@ -343,7 +345,7 @@ class AllEventsHandler( These messages make up salt's "real time" event stream. """ - log.debug("Got websocket message %s", message) + log.error("Got websocket message %s", message) if message == "websocket client ready": if self.connected: # TBD: Add ability to run commands in this branch @@ -370,7 +372,7 @@ class AllEventsHandler( def on_close(self, *args, **kwargs): """Cleanup.""" - log.debug("In the websocket close method") + log.error("In the websocket close method") self.close() def check_origin(self, origin): @@ -395,7 +397,7 @@ class FormattedEventsHandler(AllEventsHandler): # pylint: disable=W0223,W0232 These messages make up salt's "real time" event stream. """ - log.debug("Got websocket message %s", message) + log.error("Got websocket message %s", message) if message == "websocket client ready": if self.connected: # TBD: Add ability to run commands in this branch diff --git a/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py index 0850e3c371b..837bde8bfa1 100644 --- a/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py +++ b/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py @@ -27,6 +27,7 @@ class StubHandler(saltnado.BaseSaltAPIHandler): # pylint: disable=abstract-meth ret_dict[attr] = getattr(self, attr) self.write(self.serialize(ret_dict)) + self.finish() @pytest.fixture @@ -37,7 +38,7 @@ def app_urls(): ] -async def test_accept_content_type(http_client, content_type_map, subtests): +async def test_accept_content_type(http_client, content_type_map, subtests, io_loop): """ Test the base handler's accept picking """ diff --git a/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py index 7ec90651c56..a0f2f82d566 100644 --- a/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py +++ b/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py @@ -1,4 +1,5 @@ import urllib.parse +import salt.ext.tornado import pytest @@ -28,6 +29,7 @@ async def test_hook_can_handle_get_parameters(http_client, app, content_type_map ) assert response.code == 200 host = urllib.parse.urlparse(response.effective_url).netloc + print(event.fire_event.calls) event.fire_event.assert_called_once_with( { "headers": { @@ -36,7 +38,7 @@ async def test_hook_can_handle_get_parameters(http_client, app, content_type_map "Content-Type": "application/json", "Host": host, "Accept-Encoding": "gzip", - "User-Agent": "Tornado/6.1", + "User-Agent": f"Tornado/{salt.ext.tornado.version}", }, "post": {}, "get": {"param": ["1", "2"]}, diff --git a/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py index d039e75d29b..92e7d785181 100644 --- a/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py +++ b/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py @@ -16,7 +16,7 @@ pytestmark = [ @pytest.fixture -def app(client_config): +def app(client_config, io_loop): client_config.setdefault("rest_tornado", {})["websockets"] = True return rest_tornado.get_application(client_config) @@ -27,7 +27,7 @@ def http_server_port(http_server): async def test_websocket_handler_upgrade_to_websocket( - http_client, auth_creds, content_type_map, http_server_port + http_client, auth_creds, content_type_map, http_server_port, io_loop, ): response = await http_client.fetch( "/login", @@ -41,12 +41,12 @@ async def test_websocket_handler_upgrade_to_websocket( request = HTTPRequest( url, headers={"Origin": "http://example.com", "Host": "example.com"} ) - ws = await websocket_connect(request) - ws.write_message("websocket client ready") + ws = await websocket_connect(request, connect_timeout=None) + await ws.write_message("websocket client ready") ws.close() -async def test_websocket_handler_bad_token(client_config, http_server): +async def test_websocket_handler_bad_token(client_config, http_server, io_loop): """ A bad token should returns a 401 during a websocket connect """ @@ -64,7 +64,7 @@ async def test_websocket_handler_bad_token(client_config, http_server): async def test_websocket_handler_cors_origin_wildcard( - app, http_client, auth_creds, content_type_map, http_server_port + app, http_client, auth_creds, content_type_map, http_server_port, io_loop ): app.mod_opts["cors_origin"] = "*" response = await http_client.fetch( @@ -85,7 +85,7 @@ async def test_websocket_handler_cors_origin_wildcard( async def test_cors_origin_single( - app, http_client, auth_creds, content_type_map, http_server_port + app, http_client, auth_creds, content_type_map, http_server_port, io_loop ): app.mod_opts["cors_origin"] = "http://example.com" response = await http_client.fetch( @@ -116,7 +116,7 @@ async def test_cors_origin_single( async def test_cors_origin_multiple( - app, http_client, auth_creds, content_type_map, http_server_port + app, http_client, auth_creds, content_type_map, http_server_port, io_loop ): app.mod_opts["cors_origin"] = ["http://example.com", "http://foo.bar"] From f3bee7f70fd065cf73aedcb98af8bfdf53b2022c Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 15 May 2023 19:42:52 -0700 Subject: [PATCH 072/152] Remove un-needed file --- doc/topics/releases/3007.0.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 doc/topics/releases/3007.0.rst diff --git a/doc/topics/releases/3007.0.rst b/doc/topics/releases/3007.0.rst deleted file mode 100644 index e69de29bb2d..00000000000 From c86993794f65ec8a2874f6d1c92a9f4514fe355c Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 15 May 2023 20:13:37 -0700 Subject: [PATCH 073/152] Release 3006 --- salt/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/version.py b/salt/version.py index 43cb5f86f75..1962510b6da 100644 --- a/salt/version.py +++ b/salt/version.py @@ -79,7 +79,7 @@ class SaltVersionsInfo(type): SILICON = SaltVersion("Silicon" , info=3004, released=True) PHOSPHORUS = SaltVersion("Phosphorus" , info=3005, released=True) SULFUR = SaltVersion("Sulfur" , info=(3006, 0), released=True) - CHLORINE = SaltVersion("Chlorine" , info=(3007, 0)) + CHLORINE = SaltVersion("Chlorine" , info=(3007, 0), released=True) ARGON = SaltVersion("Argon" , info=(3008, 0)) POTASSIUM = SaltVersion("Potassium" , info=(3009, 0)) CALCIUM = SaltVersion("Calcium" , info=(3010, 0)) From e237d5525ecb3c24922c0b7b51dcec898cf5c334 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 15 May 2023 20:31:37 -0700 Subject: [PATCH 074/152] Fix test on windows --- tests/unit/utils/test_http.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/utils/test_http.py b/tests/unit/utils/test_http.py index 4b89cadec39..d5f89183d9f 100644 --- a/tests/unit/utils/test_http.py +++ b/tests/unit/utils/test_http.py @@ -3,6 +3,7 @@ """ import socket +import sys from contextlib import closing import pytest @@ -132,7 +133,10 @@ class HTTPTestCase(TestCase): url = "http://{host}:{port}/".format(host=host, port=port) result = http.query(url, raise_error=False) - assert result == {"error": "[Errno 111] Connection refused"}, result + if sys.platform.strtswith("win"): + assert result == {"error": "[Errno 10061] Unknown error"}, result + else: + assert result == {"error": "[Errno 111] Connection refused"}, result def test_query_error_handling(self): ret = http.query("http://127.0.0.1:0") From 7727a43cc2ac1c76ffc0eb5f89d684d2191fc8f3 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 15 May 2023 20:31:58 -0700 Subject: [PATCH 075/152] We're still generating 3006 changelogs for some reason --- doc/topics/releases/index.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/topics/releases/index.rst b/doc/topics/releases/index.rst index cf1981611d6..725f578d2f7 100644 --- a/doc/topics/releases/index.rst +++ b/doc/topics/releases/index.rst @@ -19,7 +19,7 @@ Upcoming release :maxdepth: 1 :glob: - 3007.* + 3006.* See `Install a release candidate `_ for more information about installing an RC when one is available. @@ -31,7 +31,6 @@ Previous releases :maxdepth: 1 :glob: - 3006.* 3005* 3004* 3003* From 680301504f77d58fb6647300d58de87e0b9ab8cd Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 15 May 2023 20:59:14 -0700 Subject: [PATCH 076/152] Fix multimaster test --- salt/minion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index 7a45e066d5c..97b7adc4c47 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1691,7 +1691,7 @@ class Minion(MinionBase): timeout_handler = handle_timeout # pylint: disable=unexpected-keyword-arg - self._send_req_async(load, timeout, callback=lambda f: None) + self._send_req_async(load, timeout) # pylint: enable=unexpected-keyword-arg return True From ade9da2703b5f96da33c2d5cefdbb643d8847a2c Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 15 May 2023 21:20:30 -0700 Subject: [PATCH 077/152] Fix up pre-commit and lint --- .pre-commit-config.yaml | 2 ++ salt/crypt.py | 2 +- salt/netapi/rest_tornado/__init__.py | 8 +------- salt/netapi/rest_tornado/saltnado.py | 13 +------------ salt/netapi/rest_tornado/saltnado_websockets.py | 9 +++------ .../netapi/rest_tornado/test_webhooks_handler.py | 3 ++- .../netapi/rest_tornado/test_websockets_handler.py | 6 +++++- tests/unit/utils/test_http.py | 2 +- 8 files changed, 16 insertions(+), 29 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 00d58b77e56..49b5ca4cecd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1364,6 +1364,7 @@ repos: - msgpack==1.0.3 - packaging - looseversion + - tornado - repo: https://github.com/saltstack/invoke-pre-commit rev: v1.9.0 @@ -1387,6 +1388,7 @@ repos: - msgpack==1.0.3 - packaging - looseversion + - tornado - repo: https://github.com/saltstack/invoke-pre-commit rev: v1.9.0 diff --git a/salt/crypt.py b/salt/crypt.py index 067c84200b9..79ab58d617c 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -1308,7 +1308,7 @@ class SAuth(AsyncAuth): self.authenticate() return self._crypticle - def authenticate(self, _=None): # TODO: remove unused var + def authenticate(self): # TODO: remove unused var """ Authenticate with the master, this method breaks the functional paradigm, it will update the master information from a fresh sign diff --git a/salt/netapi/rest_tornado/__init__.py b/salt/netapi/rest_tornado/__init__.py index b72b1c6bef4..ff0db64f6ea 100644 --- a/salt/netapi/rest_tornado/__init__.py +++ b/salt/netapi/rest_tornado/__init__.py @@ -39,8 +39,6 @@ def get_application(opts): except ImportError as err: log.error("ImportError! %s", err) return None - log = logging.getLogger() - log.setLevel(logging.DEBUG) mod_opts = opts.get(__virtualname__, {}) @@ -58,7 +56,6 @@ def get_application(opts): # if you have enabled websockets, add them! if mod_opts.get("websockets", False): - log.error("ENABEL WEBSOC") from . import saltnado_websockets token_pattern = r"([0-9A-Fa-f]{{{0}}})".format( @@ -76,11 +73,8 @@ def get_application(opts): (all_events_pattern, saltnado_websockets.AllEventsHandler), (formatted_events_pattern, saltnado_websockets.FormattedEventsHandler), ] - log.error("ENABEL WEBSOC - DONE") - application = salt.ext.tornado.web.Application( - paths, debug=True - ) + application = salt.ext.tornado.web.Application(paths, mod_opts.get("debug", False)) application.opts = opts application.mod_opts = mod_opts diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index e22fe7ec233..e5838afce90 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -506,11 +506,6 @@ class BaseSaltAPIHandler(salt.ext.tornado.web.RequestHandler): # pylint: disabl # TODO: set a header or something??? so we know it was a timeout self.application.event_listener.clean_by_request(self) - def finish(self): - import traceback - log.error("FINISH CALLED: %s", "\n".join(traceback.format_stack())) - super().finish() - def on_finish(self): """ When the job has been done, lets cleanup @@ -924,7 +919,6 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 """ Disbatch all lowstates to the appropriate clients """ - log.error("BEGIN DISBATCH") ret = [] # check clients before going, we want to throw 400 if one is bad @@ -961,9 +955,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 self.write(self.serialize({"return": ret})) self.finish() except RuntimeError as exc: - log.exception("DISBATCH RUNTIME ERROR") - pass # Do we need any logging here? - log.error("END DISBATCH") + log.exception("Encountered Runtime Error") @salt.ext.tornado.gen.coroutine def get_minion_returns( @@ -1431,17 +1423,14 @@ class JobsSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 """ # if you aren't authenticated, redirect to login if not self._verify_auth(): - log.error("AUTH ERROR") self.redirect("/login") return - log.error("LOWSTATE") if jid: self.lowstate = [{"fun": "jobs.list_job", "jid": jid, "client": "runner"}] else: self.lowstate = [{"fun": "jobs.list_jobs", "client": "runner"}] - log.error("DISBATCH") self.disbatch() diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py index 449de985bae..e71db6cf99d 100644 --- a/salt/netapi/rest_tornado/saltnado_websockets.py +++ b/salt/netapi/rest_tornado/saltnado_websockets.py @@ -313,20 +313,18 @@ class AllEventsHandler( """ # pylint: disable=W0221 - #@salt.ext.tornado.gen.coroutine def get(self, token): """ Check the token, returns a 401 if the token is invalid. Else open the websocket connection """ - log.error("In the websocket get method") + log.debug("In the websocket get method") self.token = token # close the connection, if not authenticated if not self.application.auth.get_tok(token): log.debug("Refusing websocket connection, bad token!") self.send_error(401) return - log.error("In the websocket get method - get") return super().get(token) def open(self, token): # pylint: disable=W0221 @@ -334,7 +332,6 @@ class AllEventsHandler( Return a websocket connection to Salt representing Salt's "real time" event stream. """ - log.error("Open websocket") self.connected = False @salt.ext.tornado.gen.coroutine @@ -345,7 +342,7 @@ class AllEventsHandler( These messages make up salt's "real time" event stream. """ - log.error("Got websocket message %s", message) + log.debug("Got websocket message %s", message) if message == "websocket client ready": if self.connected: # TBD: Add ability to run commands in this branch @@ -372,7 +369,7 @@ class AllEventsHandler( def on_close(self, *args, **kwargs): """Cleanup.""" - log.error("In the websocket close method") + log.debug("In the websocket close method") self.close() def check_origin(self, origin): diff --git a/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py index a0f2f82d566..611a548937e 100644 --- a/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py +++ b/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py @@ -1,8 +1,8 @@ import urllib.parse -import salt.ext.tornado import pytest +import salt.ext.tornado import salt.utils.json from salt.netapi.rest_tornado import saltnado from tests.support.mock import MagicMock, patch @@ -16,6 +16,7 @@ def app_urls(): async def test_hook_can_handle_get_parameters(http_client, app, content_type_map): + with patch("salt.utils.event.get_event") as get_event: with patch.dict(app.mod_opts, {"webhook_disable_auth": True}): event = MagicMock() diff --git a/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py index 92e7d785181..657f5770a3b 100644 --- a/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py +++ b/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py @@ -27,7 +27,11 @@ def http_server_port(http_server): async def test_websocket_handler_upgrade_to_websocket( - http_client, auth_creds, content_type_map, http_server_port, io_loop, + http_client, + auth_creds, + content_type_map, + http_server_port, + io_loop, ): response = await http_client.fetch( "/login", diff --git a/tests/unit/utils/test_http.py b/tests/unit/utils/test_http.py index d5f89183d9f..9f7a60ffa73 100644 --- a/tests/unit/utils/test_http.py +++ b/tests/unit/utils/test_http.py @@ -133,7 +133,7 @@ class HTTPTestCase(TestCase): url = "http://{host}:{port}/".format(host=host, port=port) result = http.query(url, raise_error=False) - if sys.platform.strtswith("win"): + if sys.platform.startswith("win"): assert result == {"error": "[Errno 10061] Unknown error"}, result else: assert result == {"error": "[Errno 111] Connection refused"}, result From 6a5e032214a1b965f7c62c630643d689252e93d2 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 16 May 2023 13:57:20 -0700 Subject: [PATCH 078/152] Deltaproxy test fix --- salt/crypt.py | 2 +- salt/metaproxy/deltaproxy.py | 58 +++++++++---------- salt/metaproxy/proxy.py | 9 ++- salt/minion.py | 4 +- .../functional/cli/test_salt_deltaproxy.py | 2 - 5 files changed, 36 insertions(+), 39 deletions(-) diff --git a/salt/crypt.py b/salt/crypt.py index 79ab58d617c..067c84200b9 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -1308,7 +1308,7 @@ class SAuth(AsyncAuth): self.authenticate() return self._crypticle - def authenticate(self): # TODO: remove unused var + def authenticate(self, _=None): # TODO: remove unused var """ Authenticate with the master, this method breaks the functional paradigm, it will update the master information from a fresh sign diff --git a/salt/metaproxy/deltaproxy.py b/salt/metaproxy/deltaproxy.py index d866d6f4c1d..55d267b1a4f 100644 --- a/salt/metaproxy/deltaproxy.py +++ b/salt/metaproxy/deltaproxy.py @@ -2,6 +2,7 @@ # Proxy minion metaproxy modules # +import asyncio import concurrent.futures import logging import os @@ -58,6 +59,7 @@ from salt.utils.process import SignalHandlingProcess, default_signals log = logging.getLogger(__name__) +@salt.ext.tornado.gen.coroutine def post_master_init(self, master): """ Function to finish init after a deltaproxy proxy @@ -337,31 +339,19 @@ def post_master_init(self, master): _failed = list() if self.opts["proxy"].get("parallel_startup"): log.debug("Initiating parallel startup for proxies") - with concurrent.futures.ThreadPoolExecutor() as executor: - futures = { - executor.submit( - subproxy_post_master_init, + waitfor = [] + for _id in self.opts["proxy"].get("ids", []): + waitfor.append( + subproxy_post_master_init( _id, uid, self.opts, self.proxy, self.utils, - ): _id - for _id in self.opts["proxy"].get("ids", []) - } - - for future in concurrent.futures.as_completed(futures): - try: - sub_proxy_data = future.result() - except Exception as exc: # pylint: disable=broad-except - _id = futures[future] - log.info( - "An exception occured during initialization for %s, skipping: %s", - _id, - exc, ) - _failed.append(_id) - continue + ) + results = yield salt.ext.tornado.gen.multi(waitfor) + for sub_proxy_data in results: minion_id = sub_proxy_data["proxy_opts"].get("id") if sub_proxy_data["proxy_minion"]: @@ -378,7 +368,7 @@ def post_master_init(self, master): log.debug("Initiating non-parallel startup for proxies") for _id in self.opts["proxy"].get("ids", []): try: - sub_proxy_data = subproxy_post_master_init( + sub_proxy_data = yield subproxy_post_master_init( _id, uid, self.opts, self.proxy, self.utils ) except Exception as exc: # pylint: disable=broad-except @@ -407,6 +397,7 @@ def post_master_init(self, master): self.ready = True +@salt.ext.tornado.gen.coroutine def subproxy_post_master_init(minion_id, uid, opts, main_proxy, main_utils): """ Function to finish init after a deltaproxy proxy @@ -415,6 +406,7 @@ def subproxy_post_master_init(minion_id, uid, opts, main_proxy, main_utils): This is primarily loading modules, pillars, etc. (since they need to know which master they connected to) for the sub proxy minions. """ + proxy_grains = {} proxy_pillar = {} @@ -433,7 +425,7 @@ def subproxy_post_master_init(minion_id, uid, opts, main_proxy, main_utils): proxy_grains = salt.loader.grains( proxyopts, proxy=main_proxy, context=proxy_context ) - proxy_pillar = salt.pillar.get_pillar( + proxy_pillar = yield salt.pillar.get_async_pillar( proxyopts, proxy_grains, minion_id, @@ -577,7 +569,9 @@ def subproxy_post_master_init(minion_id, uid, opts, main_proxy, main_utils): "__proxy_keepalive", persist=True, fire_event=False ) - return {"proxy_minion": _proxy_minion, "proxy_opts": proxyopts} + raise salt.ext.tornado.gen.Return( + {"proxy_minion": _proxy_minion, "proxy_opts": proxyopts} + ) def target(cls, minion_instance, opts, data, connected): @@ -598,11 +592,10 @@ def target(cls, minion_instance, opts, data, connected): uid = salt.utils.user.get_uid(user=opts.get("user", None)) minion_instance.proc_dir = salt.minion.get_proc_dir(opts["cachedir"], uid=uid) - with salt.ext.tornado.stack_context.StackContext(minion_instance.ctx): - if isinstance(data["fun"], tuple) or isinstance(data["fun"], list): - ProxyMinion._thread_multi_return(minion_instance, opts, data) - else: - ProxyMinion._thread_return(minion_instance, opts, data) + if isinstance(data["fun"], tuple) or isinstance(data["fun"], list): + ProxyMinion._thread_multi_return(minion_instance, opts, data) + else: + ProxyMinion._thread_return(minion_instance, opts, data) def thread_return(cls, minion_instance, opts, data): @@ -1130,7 +1123,9 @@ def tune_in(self, start=True): if self.opts["proxy"].get("parallel_startup"): with concurrent.futures.ThreadPoolExecutor() as executor: futures = [ - executor.submit(subproxy_tune_in, self.deltaproxy_objs[proxy_minion]) + executor.submit( + threaded_subproxy_tune_in, self.deltaproxy_objs[proxy_minion] + ) for proxy_minion in self.deltaproxy_objs ] @@ -1144,6 +1139,12 @@ def tune_in(self, start=True): super(ProxyMinion, self).tune_in(start=start) +def threaded_subproxy_tune_in(proxy_minion): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + subproxy_tune_in(proxy_minion) + + def subproxy_tune_in(proxy_minion, start=True): """ Tunein sub proxy minions @@ -1152,5 +1153,4 @@ def subproxy_tune_in(proxy_minion, start=True): proxy_minion.setup_beacons() proxy_minion.add_periodic_callback("cleanup", proxy_minion.cleanup_subprocesses) proxy_minion._state_run() - return proxy_minion diff --git a/salt/metaproxy/proxy.py b/salt/metaproxy/proxy.py index a399c15ef16..21566ff6453 100644 --- a/salt/metaproxy/proxy.py +++ b/salt/metaproxy/proxy.py @@ -380,11 +380,10 @@ def target(cls, minion_instance, opts, data, connected): opts["cachedir"], uid=uid ) - with salt.ext.tornado.stack_context.StackContext(minion_instance.ctx): - if isinstance(data["fun"], tuple) or isinstance(data["fun"], list): - ProxyMinion._thread_multi_return(minion_instance, opts, data) - else: - ProxyMinion._thread_return(minion_instance, opts, data) + if isinstance(data["fun"], tuple) or isinstance(data["fun"], list): + ProxyMinion._thread_multi_return(minion_instance, opts, data) + else: + ProxyMinion._thread_return(minion_instance, opts, data) def thread_return(cls, minion_instance, opts, data): diff --git a/salt/minion.py b/salt/minion.py index 97b7adc4c47..191d41e37a6 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -3820,7 +3820,7 @@ class ProxyMinion(Minion): functions. """ mp_call = _metaproxy_call(self.opts, "post_master_init") - return mp_call(self, master) + yield mp_call(self, master) @salt.ext.tornado.gen.coroutine def subproxy_post_master_init(self, minion_id, uid): @@ -3830,7 +3830,7 @@ class ProxyMinion(Minion): :rtype : None """ mp_call = _metaproxy_call(self.opts, "subproxy_post_master_init") - return mp_call(self, minion_id, uid) + yield mp_call(self, minion_id, uid) def tune_in(self, start=True): """ diff --git a/tests/pytests/functional/cli/test_salt_deltaproxy.py b/tests/pytests/functional/cli/test_salt_deltaproxy.py index b622672d575..9e099b94425 100644 --- a/tests/pytests/functional/cli/test_salt_deltaproxy.py +++ b/tests/pytests/functional/cli/test_salt_deltaproxy.py @@ -20,7 +20,6 @@ pytestmark = [ reason="Deltaproxy minions do not currently work on spawning platforms.", ), pytest.mark.core_test, - pytest.mark.skip(reason="Nest patch needs testing"), ] @@ -205,7 +204,6 @@ def test_exit_status_correct_usage_large_number_of_minions( with factory.started(): assert factory.is_running() - # Let's issue a ping the control proxy ret = salt_cli.run("test.ping", minion_tgt=proxy_minion_id) assert ret.returncode == 0 From b69152f31bd18c3628168112792285e72e9efa77 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 16 May 2023 15:26:04 -0700 Subject: [PATCH 079/152] Docstring fixes --- salt/metaproxy/deltaproxy.py | 5 +++++ salt/utils/ctx.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/metaproxy/deltaproxy.py b/salt/metaproxy/deltaproxy.py index 55d267b1a4f..8f46e2a4c2c 100644 --- a/salt/metaproxy/deltaproxy.py +++ b/salt/metaproxy/deltaproxy.py @@ -1140,6 +1140,11 @@ def tune_in(self, start=True): def threaded_subproxy_tune_in(proxy_minion): + """ + Run subproxy tune in with it's own event lopp. + + This method needs to be the target of a thread. + """ loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) subproxy_tune_in(proxy_minion) diff --git a/salt/utils/ctx.py b/salt/utils/ctx.py index 2f4b5b4c9b0..96f3474c6ef 100644 --- a/salt/utils/ctx.py +++ b/salt/utils/ctx.py @@ -14,7 +14,7 @@ request_ctxvar = contextvars.ContextVar(DEFAULT_CTX_VAR) @contextlib.contextmanager def request_context(data): """ - A context manager that sets and un-sets the loader context + A context manager that sets and un-sets the loader context. """ tok = request_ctxvar.set(data) try: @@ -24,4 +24,7 @@ def request_context(data): def get_request_context(): + """ + Get the data from the current request context. + """ return request_ctxvar.get({}) From 4672a4dcdd1b0137e443cf2077073b14df0f5b7f Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 16 May 2023 15:26:20 -0700 Subject: [PATCH 080/152] Work around for amazon linux 2 test dependency failure --- salt/metaproxy/deltaproxy.py | 2 +- tests/pytests/functional/channel/test_server.py | 5 +++++ tools/vm.py | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/metaproxy/deltaproxy.py b/salt/metaproxy/deltaproxy.py index 8f46e2a4c2c..74efd66d50e 100644 --- a/salt/metaproxy/deltaproxy.py +++ b/salt/metaproxy/deltaproxy.py @@ -1147,7 +1147,7 @@ def threaded_subproxy_tune_in(proxy_minion): """ loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - subproxy_tune_in(proxy_minion) + return subproxy_tune_in(proxy_minion) def subproxy_tune_in(proxy_minion, start=True): diff --git a/tests/pytests/functional/channel/test_server.py b/tests/pytests/functional/channel/test_server.py index bdf96679b78..6460d84a8c9 100644 --- a/tests/pytests/functional/channel/test_server.py +++ b/tests/pytests/functional/channel/test_server.py @@ -166,6 +166,11 @@ def test_pub_server_channel( req_server_channel.post_fork(handle_payload, io_loop=io_loop) if master_config["transport"] == "zeromq": p = Path(str(master_config["sock_dir"])) / "workers.ipc" + start = time.time() + while not p.exists(): + time.sleep(.3) + if time.time() - start > 20: + raise Exception("IPC socket not created") mode = os.lstat(p).st_mode assert bool(os.lstat(p).st_mode & stat.S_IRUSR) assert not bool(os.lstat(p).st_mode & stat.S_IRGRP) diff --git a/tools/vm.py b/tools/vm.py index f7b2837ae1b..70875ad42d4 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -450,6 +450,8 @@ def install_dependencies(ctx: Context, name: str, nox_session: str = "ci-test-3" Install test dependencies on VM. """ vm = VM(ctx=ctx, name=name, region_name=ctx.parser.options.region) + if name == "amazonlinux-2": + vm.run(["sudo", "yum", "install", "-y", "libffi-devel"]) returncode = vm.install_dependencies(nox_session) ctx.exit(returncode) From a2bebacc07d1427b2005a0a47fe734d043c94e2f Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 17 May 2023 00:20:19 -0700 Subject: [PATCH 081/152] Remove cruft --- salt/metaproxy/proxy.py | 2 +- tests/pytests/functional/channel/test_server.py | 2 +- .../functional/netapi/rest_tornado/test_webhooks_handler.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/metaproxy/proxy.py b/salt/metaproxy/proxy.py index 21566ff6453..1700e35027d 100644 --- a/salt/metaproxy/proxy.py +++ b/salt/metaproxy/proxy.py @@ -54,7 +54,7 @@ from salt.utils.process import SignalHandlingProcess, default_signals log = logging.getLogger(__name__) - +@salt.ext.tornado.gen.coroutine def post_master_init(self, master): """ Function to finish init after a proxy diff --git a/tests/pytests/functional/channel/test_server.py b/tests/pytests/functional/channel/test_server.py index 6460d84a8c9..86415cafa27 100644 --- a/tests/pytests/functional/channel/test_server.py +++ b/tests/pytests/functional/channel/test_server.py @@ -168,7 +168,7 @@ def test_pub_server_channel( p = Path(str(master_config["sock_dir"])) / "workers.ipc" start = time.time() while not p.exists(): - time.sleep(.3) + time.sleep(0.3) if time.time() - start > 20: raise Exception("IPC socket not created") mode = os.lstat(p).st_mode diff --git a/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py index 611a548937e..3f40c765489 100644 --- a/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py +++ b/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py @@ -30,7 +30,6 @@ async def test_hook_can_handle_get_parameters(http_client, app, content_type_map ) assert response.code == 200 host = urllib.parse.urlparse(response.effective_url).netloc - print(event.fire_event.calls) event.fire_event.assert_called_once_with( { "headers": { From 094c2c16d8ee6b7e0e094f5aaaed8e14df56be33 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 17 May 2023 00:30:25 -0700 Subject: [PATCH 082/152] Fix some windows tests --- salt/utils/asynchronous.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/utils/asynchronous.py b/salt/utils/asynchronous.py index 0c645bbc3bb..f081e43e6f1 100644 --- a/salt/utils/asynchronous.py +++ b/salt/utils/asynchronous.py @@ -19,12 +19,18 @@ def current_ioloop(io_loop): """ A context manager that will set the current ioloop to io_loop for the context """ - orig_loop = salt.ext.tornado.ioloop.IOLoop.current() + try: + orig_loop = salt.ext.tornado.ioloop.IOLoop.current() + except RuntimeError: + orig_loop = None io_loop.make_current() try: yield finally: - orig_loop.make_current() + if orig_loop: + orig_loop.make_current() + else: + io_loop.clear_current() class SyncWrapper: From ca241b75d42226fb85d98e7df7f3c6d005ee3a39 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 17 May 2023 01:04:14 -0700 Subject: [PATCH 083/152] Add back reporting of failed sub-proxies --- salt/metaproxy/deltaproxy.py | 9 ++++++++- salt/metaproxy/proxy.py | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/metaproxy/deltaproxy.py b/salt/metaproxy/deltaproxy.py index 74efd66d50e..058f410ae58 100644 --- a/salt/metaproxy/deltaproxy.py +++ b/salt/metaproxy/deltaproxy.py @@ -350,9 +350,16 @@ def post_master_init(self, master): self.utils, ) ) - results = yield salt.ext.tornado.gen.multi(waitfor) + + try: + results = yield salt.ext.tornado.gen.multi(waitfor) + except Exception as exc: # pylint: disable=broad-except + log.error("Errors loading sub proxies") + + _failed = self.opts["proxy"].get("ids", []) for sub_proxy_data in results: minion_id = sub_proxy_data["proxy_opts"].get("id") + _failed.remove(minion_id) if sub_proxy_data["proxy_minion"]: self.deltaproxy_opts[minion_id] = sub_proxy_data["proxy_opts"] diff --git a/salt/metaproxy/proxy.py b/salt/metaproxy/proxy.py index 1700e35027d..039081cf5f1 100644 --- a/salt/metaproxy/proxy.py +++ b/salt/metaproxy/proxy.py @@ -54,6 +54,7 @@ from salt.utils.process import SignalHandlingProcess, default_signals log = logging.getLogger(__name__) + @salt.ext.tornado.gen.coroutine def post_master_init(self, master): """ From 486802f6dd5edb4908e3dde7f8438a4b1399599b Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 18 May 2023 00:10:57 -0700 Subject: [PATCH 084/152] Fix more deltaproxy tests --- salt/__init__.py | 1 + salt/metaproxy/deltaproxy.py | 5 +++-- salt/minion.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/__init__.py b/salt/__init__.py index 49fea82bb42..58f756d96ce 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -12,6 +12,7 @@ if sys.version_info < (3,): ) sys.stderr.flush() + USE_VENDORED_TORNADO = False diff --git a/salt/metaproxy/deltaproxy.py b/salt/metaproxy/deltaproxy.py index 058f410ae58..de07a041956 100644 --- a/salt/metaproxy/deltaproxy.py +++ b/salt/metaproxy/deltaproxy.py @@ -356,10 +356,11 @@ def post_master_init(self, master): except Exception as exc: # pylint: disable=broad-except log.error("Errors loading sub proxies") - _failed = self.opts["proxy"].get("ids", []) + _failed = self.opts["proxy"].get("ids", [])[:] for sub_proxy_data in results: minion_id = sub_proxy_data["proxy_opts"].get("id") - _failed.remove(minion_id) + if minion_id in _failed: + _failed.remove(minion_id) if sub_proxy_data["proxy_minion"]: self.deltaproxy_opts[minion_id] = sub_proxy_data["proxy_opts"] diff --git a/salt/minion.py b/salt/minion.py index 191d41e37a6..5bceada8b53 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1162,6 +1162,7 @@ class MinionManager(MinionBase): minion.opts["master"], exc_info=True, ) + break # Multi Master Tune In def tune_in(self): From 9dcbea0d4b6642599cbe60a3b8c26d560b9bc93e Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 18 May 2023 01:37:13 -0700 Subject: [PATCH 085/152] Windows fix --- salt/minion.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/minion.py b/salt/minion.py index 5bceada8b53..dc151643543 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1899,6 +1899,9 @@ class Minion(MinionBase): This method should be used as a threading target, start the actual minion side execution. """ + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + minion_instance.gen_modules() fn_ = os.path.join(minion_instance.proc_dir, data["jid"]) @@ -2085,6 +2088,9 @@ class Minion(MinionBase): This method should be used as a threading target, start the actual minion side execution. """ + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + minion_instance.gen_modules() fn_ = os.path.join(minion_instance.proc_dir, data["jid"]) From e87bc38389b777132e2c1d7cb43079fc7f25b5f5 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 18 May 2023 02:35:10 -0700 Subject: [PATCH 086/152] Try without the loop policy --- salt/minion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/minion.py b/salt/minion.py index dc151643543..64291459e4a 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1,6 +1,7 @@ """ Routines to set up a minion """ +import asyncio import binascii import contextlib import copy From 4baea1a97be0389fabe5307d084579134a1f9b7a Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 17 May 2023 04:19:07 -0700 Subject: [PATCH 087/152] Remove vendored tornado --- .pre-commit-config.yaml | 2 +- salt/__init__.py | 34 - salt/channel/client.py | 51 +- salt/channel/server.py | 33 +- salt/client/__init__.py | 15 +- salt/crypt.py | 23 +- salt/engines/ircbot.py | 17 +- salt/engines/webhook.py | 19 +- salt/ext/tornado/__init__.py | 30 - salt/ext/tornado/_locale_data.py | 86 - salt/ext/tornado/auth.py | 1155 ------ salt/ext/tornado/autoreload.py | 335 -- salt/ext/tornado/concurrent.py | 547 --- salt/ext/tornado/curl_httpclient.py | 515 --- salt/ext/tornado/escape.py | 399 -- salt/ext/tornado/gen.py | 1304 ------- salt/ext/tornado/http1connection.py | 743 ---- salt/ext/tornado/httpclient.py | 679 ---- salt/ext/tornado/httpserver.py | 326 -- salt/ext/tornado/httputil.py | 1058 ------ salt/ext/tornado/ioloop.py | 1043 ------ salt/ext/tornado/iostream.py | 1569 -------- salt/ext/tornado/locale.py | 522 --- salt/ext/tornado/locks.py | 513 --- salt/ext/tornado/log.py | 291 -- salt/ext/tornado/netutil.py | 530 --- salt/ext/tornado/options.py | 595 --- salt/ext/tornado/platform/__init__.py | 1 - salt/ext/tornado/platform/asyncio.py | 223 -- salt/ext/tornado/platform/auto.py | 60 - salt/ext/tornado/platform/auto.pyi | 4 - salt/ext/tornado/platform/caresresolver.py | 80 - salt/ext/tornado/platform/common.py | 114 - salt/ext/tornado/platform/epoll.py | 27 - salt/ext/tornado/platform/interface.py | 68 - salt/ext/tornado/platform/kqueue.py | 92 - salt/ext/tornado/platform/posix.py | 71 - salt/ext/tornado/platform/select.py | 77 - salt/ext/tornado/platform/twisted.py | 592 --- salt/ext/tornado/platform/windows.py | 21 - salt/ext/tornado/process.py | 366 -- salt/ext/tornado/queues.py | 367 -- salt/ext/tornado/routing.py | 626 ---- salt/ext/tornado/simple_httpclient.py | 568 --- salt/ext/tornado/speedups.c | 52 - salt/ext/tornado/speedups.pyi | 1 - salt/ext/tornado/stack_context.py | 391 -- salt/ext/tornado/tcpclient.py | 225 -- salt/ext/tornado/tcpserver.py | 301 -- salt/ext/tornado/template.py | 979 ----- salt/ext/tornado/test/__init__.py | 1 - salt/ext/tornado/test/__main__.py | 15 - salt/ext/tornado/test/asyncio_test.py | 121 - salt/ext/tornado/test/auth_test.py | 548 --- salt/ext/tornado/test/concurrent_test.py | 436 --- .../tornado/test/csv_translations/fr_FR.csv | 1 - salt/ext/tornado/test/curl_httpclient_test.py | 135 - salt/ext/tornado/test/escape_test.py | 246 -- salt/ext/tornado/test/gen_test.py | 1468 -------- .../test/gettext_translations/extract_me.py | 17 - .../fr_FR/LC_MESSAGES/tornado_test.po | 47 - salt/ext/tornado/test/http1connection_test.py | 62 - salt/ext/tornado/test/httpclient_test.py | 686 ---- salt/ext/tornado/test/httpserver_test.py | 1135 ------ salt/ext/tornado/test/httputil_test.py | 467 --- salt/ext/tornado/test/import_test.py | 48 - salt/ext/tornado/test/ioloop_test.py | 682 ---- salt/ext/tornado/test/iostream_test.py | 1144 ------ salt/ext/tornado/test/locale_test.py | 131 - salt/ext/tornado/test/locks_test.py | 519 --- salt/ext/tornado/test/log_test.py | 242 -- salt/ext/tornado/test/netutil_test.py | 216 -- salt/ext/tornado/test/options_test.cfg | 7 - salt/ext/tornado/test/options_test.py | 294 -- salt/ext/tornado/test/process_test.py | 264 -- salt/ext/tornado/test/queues_test.py | 424 --- salt/ext/tornado/test/resolve_test_helper.py | 12 - salt/ext/tornado/test/routing_test.py | 225 -- salt/ext/tornado/test/runtests.py | 191 - .../tornado/test/simple_httpclient_test.py | 778 ---- salt/ext/tornado/test/stack_context_test.py | 290 -- salt/ext/tornado/test/static/dir/index.html | 1 - salt/ext/tornado/test/static/robots.txt | 2 - salt/ext/tornado/test/static/sample.xml | 23 - salt/ext/tornado/test/static/sample.xml.bz2 | Bin 285 -> 0 bytes salt/ext/tornado/test/static/sample.xml.gz | Bin 264 -> 0 bytes salt/ext/tornado/test/static_foo.txt | 2 - salt/ext/tornado/test/tcpclient_test.py | 314 -- salt/ext/tornado/test/tcpserver_test.py | 71 - salt/ext/tornado/test/template_test.py | 497 --- salt/ext/tornado/test/templates/utf8.html | 1 - salt/ext/tornado/test/test.crt | 15 - salt/ext/tornado/test/test.key | 16 - salt/ext/tornado/test/testing_test.py | 279 -- salt/ext/tornado/test/twisted_test.py | 732 ---- salt/ext/tornado/test/util.py | 135 - salt/ext/tornado/test/util_test.py | 228 -- salt/ext/tornado/test/web_test.py | 2890 --------------- salt/ext/tornado/test/websocket_test.py | 639 ---- salt/ext/tornado/test/windows_test.py | 28 - salt/ext/tornado/test/wsgi_test.py | 104 - salt/ext/tornado/testing.py | 743 ---- salt/ext/tornado/util.py | 476 --- salt/ext/tornado/web.py | 3287 ----------------- salt/ext/tornado/websocket.py | 1245 ------- salt/ext/tornado/wsgi.py | 359 -- salt/fileclient.py | 7 +- salt/master.py | 9 +- salt/metaproxy/deltaproxy.py | 17 +- salt/metaproxy/proxy.py | 9 +- salt/minion.py | 79 +- salt/netapi/rest_tornado/__init__.py | 14 +- salt/netapi/rest_tornado/saltnado.py | 93 +- .../rest_tornado/saltnado_websockets.py | 11 +- salt/pillar/__init__.py | 11 +- salt/transport/base.py | 6 +- salt/transport/ipc.py | 89 +- salt/transport/tcp.py | 111 +- salt/transport/zeromq.py | 40 +- salt/utils/asynchronous.py | 8 +- salt/utils/event.py | 21 +- salt/utils/gitfs.py | 5 +- salt/utils/http.py | 21 +- salt/utils/process.py | 3 +- salt/utils/thin.py | 2 +- tests/integration/minion/test_minion_cache.py | 2 +- tests/integration/modules/test_gem.py | 2 +- .../netapi/rest_tornado/test_app.py | 14 +- tests/pytests/conftest.py | 8 +- .../pytests/functional/channel/test_server.py | 8 +- .../netapi/rest_cherrypy/conftest.py | 4 +- .../netapi/rest_cherrypy/test_auth.py | 2 +- .../netapi/rest_cherrypy/test_auth_pam.py | 3 +- .../netapi/rest_cherrypy/test_out_formats.py | 3 +- .../netapi/rest_tornado/test_auth_handler.py | 2 +- .../rest_tornado/test_auth_handler_pam.py | 2 +- .../rest_tornado/test_base_api_handler.py | 2 +- .../netapi/rest_tornado/test_utils.py | 5 +- .../rest_tornado/test_webhooks_handler.py | 4 +- .../rest_tornado/test_websockets_handler.py | 4 +- .../functional/transport/ipc/test_client.py | 2 +- .../transport/ipc/test_subscriber.py | 8 +- .../transport/server/test_req_channel.py | 10 +- .../transport/tcp/test_message_client.py | 18 +- .../utils/test_async_event_publisher.py | 1 - .../netapi/rest_cherrypy/conftest.py | 4 +- .../netapi/rest_cherrypy/test_auth.py | 2 +- .../netapi/rest_cherrypy/test_run.py | 3 +- .../rest_tornado/test_events_api_handler.py | 4 +- .../rest_tornado/test_minions_api_handler.py | 2 +- .../netapi/rest_tornado/test_root_handler.py | 2 +- .../unit/fileserver/gitfs/test_gitfs.py | 6 +- .../fileserver/gitfs/test_gitfs_config.py | 6 +- tests/pytests/unit/test_ext_importers.py | 54 - tests/pytests/unit/test_minion.py | 56 +- tests/pytests/unit/transport/test_ipc.py | 4 +- tests/pytests/unit/transport/test_tcp.py | 30 +- tests/pytests/unit/transport/test_zeromq.py | 44 +- tests/pytests/unit/utils/event/test_event.py | 14 +- .../unit/utils/event/test_event_return.py | 1 - tests/support/helpers.py | 24 +- tests/support/netapi.py | 16 +- tests/support/pytest/transport.py | 20 +- tests/unit/modules/test_random_org.py | 2 +- .../unit/netapi/rest_tornado/test_saltnado.py | 315 +- tests/unit/test_proxy_minion.py | 10 +- tests/unit/transport/mixins.py | 6 +- tests/unit/transport/test_ipc.py | 12 +- tests/unit/transport/test_tcp.py | 10 +- tests/unit/utils/test_asynchronous.py | 21 +- tests/unit/utils/test_context.py | 1 - tests/unit/utils/test_gitfs.py | 5 +- tests/unit/utils/test_http.py | 2 +- 173 files changed, 704 insertions(+), 41311 deletions(-) delete mode 100644 salt/ext/tornado/__init__.py delete mode 100644 salt/ext/tornado/_locale_data.py delete mode 100644 salt/ext/tornado/auth.py delete mode 100644 salt/ext/tornado/autoreload.py delete mode 100644 salt/ext/tornado/concurrent.py delete mode 100644 salt/ext/tornado/curl_httpclient.py delete mode 100644 salt/ext/tornado/escape.py delete mode 100644 salt/ext/tornado/gen.py delete mode 100644 salt/ext/tornado/http1connection.py delete mode 100644 salt/ext/tornado/httpclient.py delete mode 100644 salt/ext/tornado/httpserver.py delete mode 100644 salt/ext/tornado/httputil.py delete mode 100644 salt/ext/tornado/ioloop.py delete mode 100644 salt/ext/tornado/iostream.py delete mode 100644 salt/ext/tornado/locale.py delete mode 100644 salt/ext/tornado/locks.py delete mode 100644 salt/ext/tornado/log.py delete mode 100644 salt/ext/tornado/netutil.py delete mode 100644 salt/ext/tornado/options.py delete mode 100644 salt/ext/tornado/platform/__init__.py delete mode 100644 salt/ext/tornado/platform/asyncio.py delete mode 100644 salt/ext/tornado/platform/auto.py delete mode 100644 salt/ext/tornado/platform/auto.pyi delete mode 100644 salt/ext/tornado/platform/caresresolver.py delete mode 100644 salt/ext/tornado/platform/common.py delete mode 100644 salt/ext/tornado/platform/epoll.py delete mode 100644 salt/ext/tornado/platform/interface.py delete mode 100644 salt/ext/tornado/platform/kqueue.py delete mode 100644 salt/ext/tornado/platform/posix.py delete mode 100644 salt/ext/tornado/platform/select.py delete mode 100644 salt/ext/tornado/platform/twisted.py delete mode 100644 salt/ext/tornado/platform/windows.py delete mode 100644 salt/ext/tornado/process.py delete mode 100644 salt/ext/tornado/queues.py delete mode 100644 salt/ext/tornado/routing.py delete mode 100644 salt/ext/tornado/simple_httpclient.py delete mode 100644 salt/ext/tornado/speedups.c delete mode 100644 salt/ext/tornado/speedups.pyi delete mode 100644 salt/ext/tornado/stack_context.py delete mode 100644 salt/ext/tornado/tcpclient.py delete mode 100644 salt/ext/tornado/tcpserver.py delete mode 100644 salt/ext/tornado/template.py delete mode 100644 salt/ext/tornado/test/__init__.py delete mode 100644 salt/ext/tornado/test/__main__.py delete mode 100644 salt/ext/tornado/test/asyncio_test.py delete mode 100644 salt/ext/tornado/test/auth_test.py delete mode 100644 salt/ext/tornado/test/concurrent_test.py delete mode 100644 salt/ext/tornado/test/csv_translations/fr_FR.csv delete mode 100644 salt/ext/tornado/test/curl_httpclient_test.py delete mode 100644 salt/ext/tornado/test/escape_test.py delete mode 100644 salt/ext/tornado/test/gen_test.py delete mode 100644 salt/ext/tornado/test/gettext_translations/extract_me.py delete mode 100644 salt/ext/tornado/test/gettext_translations/fr_FR/LC_MESSAGES/tornado_test.po delete mode 100644 salt/ext/tornado/test/http1connection_test.py delete mode 100644 salt/ext/tornado/test/httpclient_test.py delete mode 100644 salt/ext/tornado/test/httpserver_test.py delete mode 100644 salt/ext/tornado/test/httputil_test.py delete mode 100644 salt/ext/tornado/test/import_test.py delete mode 100644 salt/ext/tornado/test/ioloop_test.py delete mode 100644 salt/ext/tornado/test/iostream_test.py delete mode 100644 salt/ext/tornado/test/locale_test.py delete mode 100644 salt/ext/tornado/test/locks_test.py delete mode 100644 salt/ext/tornado/test/log_test.py delete mode 100644 salt/ext/tornado/test/netutil_test.py delete mode 100644 salt/ext/tornado/test/options_test.cfg delete mode 100644 salt/ext/tornado/test/options_test.py delete mode 100644 salt/ext/tornado/test/process_test.py delete mode 100644 salt/ext/tornado/test/queues_test.py delete mode 100644 salt/ext/tornado/test/resolve_test_helper.py delete mode 100644 salt/ext/tornado/test/routing_test.py delete mode 100644 salt/ext/tornado/test/runtests.py delete mode 100644 salt/ext/tornado/test/simple_httpclient_test.py delete mode 100644 salt/ext/tornado/test/stack_context_test.py delete mode 100644 salt/ext/tornado/test/static/dir/index.html delete mode 100644 salt/ext/tornado/test/static/robots.txt delete mode 100644 salt/ext/tornado/test/static/sample.xml delete mode 100644 salt/ext/tornado/test/static/sample.xml.bz2 delete mode 100644 salt/ext/tornado/test/static/sample.xml.gz delete mode 100644 salt/ext/tornado/test/static_foo.txt delete mode 100644 salt/ext/tornado/test/tcpclient_test.py delete mode 100644 salt/ext/tornado/test/tcpserver_test.py delete mode 100644 salt/ext/tornado/test/template_test.py delete mode 100644 salt/ext/tornado/test/templates/utf8.html delete mode 100644 salt/ext/tornado/test/test.crt delete mode 100644 salt/ext/tornado/test/test.key delete mode 100644 salt/ext/tornado/test/testing_test.py delete mode 100644 salt/ext/tornado/test/twisted_test.py delete mode 100644 salt/ext/tornado/test/util.py delete mode 100644 salt/ext/tornado/test/util_test.py delete mode 100644 salt/ext/tornado/test/web_test.py delete mode 100644 salt/ext/tornado/test/websocket_test.py delete mode 100644 salt/ext/tornado/test/windows_test.py delete mode 100644 salt/ext/tornado/test/wsgi_test.py delete mode 100644 salt/ext/tornado/testing.py delete mode 100644 salt/ext/tornado/util.py delete mode 100644 salt/ext/tornado/web.py delete mode 100644 salt/ext/tornado/websocket.py delete mode 100644 salt/ext/tornado/wsgi.py delete mode 100644 tests/pytests/unit/test_ext_importers.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 49b5ca4cecd..d8df181635d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1235,7 +1235,7 @@ repos: alias: rewrite-tests name: Rewrite Salt's Test Suite files: ^tests/.*\.py$ - args: [--silent, -E, fix_asserts, -E, fix_docstrings] + args: [--silent, -E, fix_asserts, -E, fix_docstrings, -E, fix_tornado_imports] - repo: https://github.com/timothycrosley/isort rev: 5.12.0 diff --git a/salt/__init__.py b/salt/__init__.py index 58f756d96ce..485b17553c1 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -12,40 +12,6 @@ if sys.version_info < (3,): ) sys.stderr.flush() - -USE_VENDORED_TORNADO = False - - -class TornadoImporter: - def find_module(self, module_name, package_path=None): - if USE_VENDORED_TORNADO: - if module_name.startswith("tornado"): - return self - else: - if module_name.startswith("salt.ext.tornado"): - return self - return None - - def load_module(self, name): - if USE_VENDORED_TORNADO: - mod = importlib.import_module("salt.ext.{}".format(name)) - else: - # Remove 'salt.ext.' from the module - mod = importlib.import_module(name[9:]) - sys.modules[name] = mod - return mod - - def create_module(self, spec): - return self.load_module(spec.name) - - def exec_module(self, module): - return None - - -# Try our importer first -sys.meta_path = [TornadoImporter()] + sys.meta_path - - # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( "once", # Show once diff --git a/salt/channel/client.py b/salt/channel/client.py index e5b073ccdba..da09f78fb83 100644 --- a/salt/channel/client.py +++ b/salt/channel/client.py @@ -10,10 +10,11 @@ import os import time import uuid +import tornado.gen +import tornado.ioloop + import salt.crypt import salt.exceptions -import salt.ext.tornado.gen -import salt.ext.tornado.ioloop import salt.payload import salt.transport.frame import salt.utils.event @@ -119,7 +120,7 @@ class AsyncReqChannel: opts["master_uri"] = kwargs["master_uri"] io_loop = kwargs.get("io_loop") if io_loop is None: - io_loop = salt.ext.tornado.ioloop.IOLoop.current() + io_loop = tornado.ioloop.IOLoop.current() crypt = kwargs.get("crypt", "aes") if crypt != "clear": @@ -157,7 +158,7 @@ class AsyncReqChannel: "version": 2, } - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def crypted_transfer_decode_dictentry( self, load, @@ -204,12 +205,12 @@ class AsyncReqChannel: # Validate the nonce. if data["nonce"] != nonce: raise salt.crypt.AuthenticationError("Pillar nonce verification failed.") - raise salt.ext.tornado.gen.Return(data["pillar"]) + raise tornado.gen.Return(data["pillar"]) def verify_signature(self, data, sig): return salt.crypt.verify_signature(self.master_pubkey_path, data, sig) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _crypted_transfer(self, load, timeout=60, raw=False): """ Send a load across the wire, with encryption @@ -227,7 +228,7 @@ class AsyncReqChannel: if load and isinstance(load, dict): load["nonce"] = nonce - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _do_transfer(): # Yield control to the caller. When send() completes, resume by populating data with the Future.result data = yield self.transport.send( @@ -242,7 +243,7 @@ class AsyncReqChannel: data = self.auth.crypticle.loads(data, raw, nonce=nonce) if not raw or self.ttype == "tcp": # XXX Why is this needed for tcp data = salt.transport.frame.decode_embedded_strs(data) - raise salt.ext.tornado.gen.Return(data) + raise tornado.gen.Return(data) if not self.auth.authenticated: # Return control back to the caller, resume when authentication succeeds @@ -254,9 +255,9 @@ class AsyncReqChannel: # If auth error, return control back to the caller, continue when authentication succeeds yield self.auth.authenticate() ret = yield _do_transfer() - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _uncrypted_transfer(self, load, timeout=60): """ Send a load across the wire in cleartext @@ -269,13 +270,13 @@ class AsyncReqChannel: timeout=timeout, ) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def connect(self): yield self.transport.connect() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send(self, load, tries=3, timeout=60, raw=False): """ Send a request, return a future which will complete when we send the message @@ -301,7 +302,7 @@ class AsyncReqChannel: else: _try += 1 continue - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) def close(self): """ @@ -355,7 +356,7 @@ class AsyncPubChannel: io_loop = kwargs.get("io_loop") if io_loop is None: - io_loop = salt.ext.tornado.ioloop.IOLoop.current() + io_loop = tornado.ioloop.IOLoop.current() auth = salt.crypt.AsyncAuth(opts, io_loop=io_loop) transport = salt.transport.publish_client(opts, io_loop) @@ -376,7 +377,7 @@ class AsyncPubChannel: def crypt(self): return "aes" if self.auth else "clear" - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def connect(self): """ Return a future which completes when connected to the remote publisher @@ -420,7 +421,7 @@ class AsyncPubChannel: if callback is None: return self.transport.on_recv(None) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def wrap_callback(messages): payload = yield self.transport._decode_messages(messages) decoded = yield self._decode_payload(payload) @@ -437,7 +438,7 @@ class AsyncPubChannel: "version": 2, } - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send_id(self, tok, force_auth): """ Send the minion id to the master so that the master may better @@ -447,13 +448,13 @@ class AsyncPubChannel: """ load = {"id": self.opts["id"], "tok": tok} - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _do_transfer(): msg = self._package_load(self.auth.crypticle.dumps(load)) package = salt.transport.frame.frame_msg(msg, header=None) yield self.transport.send(package) - raise salt.ext.tornado.gen.Return(True) + raise tornado.gen.Return(True) if force_auth or not self.auth.authenticated: count = 0 @@ -469,13 +470,13 @@ class AsyncPubChannel: count += 1 try: ret = yield _do_transfer() - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) except salt.crypt.AuthenticationError: yield self.auth.authenticate() ret = yield _do_transfer() - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def connect_callback(self, result): if self._closing: return @@ -547,7 +548,7 @@ class AsyncPubChannel: "Message signature failed to validate." ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _decode_payload(self, payload): # we need to decrypt it log.trace("Decoding payload: %s", payload) @@ -559,7 +560,7 @@ class AsyncPubChannel: yield self.auth.authenticate() payload["load"] = self.auth.crypticle.loads(payload["load"]) - raise salt.ext.tornado.gen.Return(payload) + raise tornado.gen.Return(payload) def __enter__(self): return self diff --git a/salt/channel/server.py b/salt/channel/server.py index a2117f2934d..7627a30de43 100644 --- a/salt/channel/server.py +++ b/salt/channel/server.py @@ -10,8 +10,9 @@ import logging import os import shutil +import tornado.gen + import salt.crypt -import salt.ext.tornado.gen import salt.master import salt.payload import salt.transport.frame @@ -100,7 +101,7 @@ class ReqServerChannel: if hasattr(self.transport, "post_fork"): self.transport.post_fork(self.handle_message, io_loop) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def handle_message(self, payload): try: payload = self._decode_payload(payload) @@ -115,7 +116,7 @@ class ReqServerChannel: ) else: log.error("Bad load from minion: %s: %s", exc_type, exc) - raise salt.ext.tornado.gen.Return("bad load") + raise tornado.gen.Return("bad load") # TODO helper functions to normalize payload? if not isinstance(payload, dict) or not isinstance(payload.get("load"), dict): @@ -124,18 +125,16 @@ class ReqServerChannel: payload, payload.get("load"), ) - raise salt.ext.tornado.gen.Return("payload and load must be a dict") + raise tornado.gen.Return("payload and load must be a dict") try: id_ = payload["load"].get("id", "") if "\0" in id_: log.error("Payload contains an id with a null byte: %s", payload) - raise salt.ext.tornado.gen.Return("bad load: id contains a null byte") + raise tornado.gen.Return("bad load: id contains a null byte") except TypeError: log.error("Payload contains non-string id: %s", payload) - raise salt.ext.tornado.gen.Return( - "bad load: id {} is not a string".format(id_) - ) + raise tornado.gen.Return("bad load: id {} is not a string".format(id_)) version = 0 if "version" in payload: @@ -148,9 +147,7 @@ class ReqServerChannel: # intercept the "_auth" commands, since the main daemon shouldn't know # anything about our key auth if payload["enc"] == "clear" and payload.get("load", {}).get("cmd") == "_auth": - raise salt.ext.tornado.gen.Return( - self._auth(payload["load"], sign_messages) - ) + raise tornado.gen.Return(self._auth(payload["load"], sign_messages)) nonce = None if version > 1: @@ -164,15 +161,15 @@ class ReqServerChannel: except Exception as e: # pylint: disable=broad-except # always attempt to return an error to the minion log.error("Some exception handling a payload from minion", exc_info=True) - raise salt.ext.tornado.gen.Return("Some exception handling minion payload") + raise tornado.gen.Return("Some exception handling minion payload") req_fun = req_opts.get("fun", "send") if req_fun == "send_clear": - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) elif req_fun == "send": - raise salt.ext.tornado.gen.Return(self.crypticle.dumps(ret, nonce)) + raise tornado.gen.Return(self.crypticle.dumps(ret, nonce)) elif req_fun == "send_private": - raise salt.ext.tornado.gen.Return( + raise tornado.gen.Return( self._encrypt_private( ret, req_opts["key"], @@ -183,7 +180,7 @@ class ReqServerChannel: ) log.error("Unknown req_fun %s", req_fun) # always attempt to return an error to the minion - raise salt.ext.tornado.gen.Return("Server-side exception handling payload") + raise tornado.gen.Return("Server-side exception handling payload") def _encrypt_private(self, ret, dictkey, target, nonce=None, sign_messages=True): """ @@ -840,7 +837,7 @@ class PubServerChannel: data, salt.utils.event.tagify("present", "presence") ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def publish_payload(self, load, *args): unpacked_package = self.wrap_payload(load) try: @@ -853,7 +850,7 @@ class PubServerChannel: ret = yield self.transport.publish_payload(payload, topic_list) else: ret = yield self.transport.publish_payload(payload) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) def wrap_payload(self, load): payload = {"enc": "aes"} diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 7ce8963b8f6..39a8b33268d 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -25,11 +25,12 @@ import sys import time from datetime import datetime +import tornado.gen + import salt.cache import salt.channel.client import salt.config import salt.defaults.exitcodes -import salt.ext.tornado.gen import salt.loader import salt.payload import salt.syspaths as syspaths @@ -416,7 +417,7 @@ class LocalClient: ) return _res["minions"] - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def run_job_async( self, tgt, @@ -473,7 +474,7 @@ class LocalClient: # Convert to generic client error and pass along message raise SaltClientError(general_exception) - raise salt.ext.tornado.gen.Return(self._check_pub_data(pub_data, listen=listen)) + raise tornado.gen.Return(self._check_pub_data(pub_data, listen=listen)) def cmd_async( self, tgt, fun, arg=(), tgt_type="glob", ret="", jid="", kwarg=None, **kwargs @@ -1941,7 +1942,7 @@ class LocalClient: return {"jid": payload["load"]["jid"], "minions": payload["load"]["minions"]} - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def pub_async( self, tgt, @@ -2022,7 +2023,7 @@ class LocalClient: # and try again if the key has changed key = self.__read_master_key() if key == self.key: - raise salt.ext.tornado.gen.Return(payload) + raise tornado.gen.Return(payload) self.key = key payload_kwargs["key"] = self.key payload = yield channel.send(payload_kwargs) @@ -2040,9 +2041,9 @@ class LocalClient: raise PublishError(error) if not payload: - raise salt.ext.tornado.gen.Return(payload) + raise tornado.gen.Return(payload) - raise salt.ext.tornado.gen.Return( + raise tornado.gen.Return( {"jid": payload["load"]["jid"], "minions": payload["load"]["minions"]} ) diff --git a/salt/crypt.py b/salt/crypt.py index 067c84200b9..452c86bbdf3 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -20,9 +20,10 @@ import traceback import uuid import weakref +import tornado.gen + import salt.channel.client import salt.defaults.exitcodes -import salt.ext.tornado.gen import salt.payload import salt.utils.crypt import salt.utils.decorators @@ -495,7 +496,7 @@ class AsyncAuth: Only create one instance of AsyncAuth per __key() """ # do we have any mapping for this io_loop - io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current() + io_loop = io_loop or tornado.ioloop.IOLoop.current() if io_loop not in AsyncAuth.instance_map: AsyncAuth.instance_map[io_loop] = weakref.WeakValueDictionary() loop_instance_map = AsyncAuth.instance_map[io_loop] @@ -546,7 +547,7 @@ class AsyncAuth: if not os.path.isfile(self.pub_path): self.get_keys() - self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current() + self.io_loop = io_loop or tornado.ioloop.IOLoop.current() salt.utils.crypt.reinit_crypto() key = self.__key(self.opts) @@ -555,7 +556,7 @@ class AsyncAuth: creds = AsyncAuth.creds_map[key] self._creds = creds self._crypticle = Crypticle(self.opts, creds["aes"]) - self._authenticate_future = salt.ext.tornado.concurrent.Future() + self._authenticate_future = tornado.concurrent.Future() self._authenticate_future.set_result(True) else: self.authenticate() @@ -610,7 +611,7 @@ class AsyncAuth: ): future = self._authenticate_future else: - future = salt.ext.tornado.concurrent.Future() + future = tornado.concurrent.Future() self._authenticate_future = future self.io_loop.add_callback(self._authenticate) @@ -624,7 +625,7 @@ class AsyncAuth: return future - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _authenticate(self): """ Authenticate with the master, this method breaks the functional @@ -675,7 +676,7 @@ class AsyncAuth: log.info( "Waiting %s seconds before retry.", acceptance_wait_time ) - yield salt.ext.tornado.gen.sleep(acceptance_wait_time) + yield tornado.gen.sleep(acceptance_wait_time) if acceptance_wait_time < acceptance_wait_time_max: acceptance_wait_time += acceptance_wait_time log.debug( @@ -721,7 +722,7 @@ class AsyncAuth: salt.utils.event.tagify(prefix="auth", suffix="creds"), ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def sign_in(self, timeout=60, safe=True, tries=1, channel=None): """ Send a sign in request to the master, sets the key information and @@ -762,9 +763,9 @@ class AsyncAuth: except SaltReqTimeoutError as e: if safe: log.warning("SaltReqTimeoutError: %s", e) - raise salt.ext.tornado.gen.Return("retry") + raise tornado.gen.Return("retry") if self.opts.get("detect_mode") is True: - raise salt.ext.tornado.gen.Return("retry") + raise tornado.gen.Return("retry") else: raise SaltClientError( "Attempt to authenticate with the salt master failed with timeout" @@ -774,7 +775,7 @@ class AsyncAuth: if close_channel: channel.close() ret = self.handle_signin_response(sign_in_payload, payload) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) def handle_signin_response(self, sign_in_payload, payload): auth = {} diff --git a/salt/engines/ircbot.py b/salt/engines/ircbot.py index 1dab78dbbc5..6126b2c1076 100644 --- a/salt/engines/ircbot.py +++ b/salt/engines/ircbot.py @@ -61,8 +61,9 @@ import socket import ssl from collections import namedtuple -import salt.ext.tornado.ioloop -import salt.ext.tornado.iostream +import tornado.ioloop +import tornado.iostream + import salt.utils.event log = logging.getLogger(__name__) @@ -101,18 +102,18 @@ class IRCClient: self.allow_hosts = allow_hosts self.allow_nicks = allow_nicks self.disable_query = disable_query - self.io_loop = salt.ext.tornado.ioloop.IOLoop(make_current=False) + self.io_loop = tornado.ioloop.IOLoop(make_current=False) self.io_loop.make_current() self._connect() def _connect(self): _sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) if self.ssl is True: - self._stream = salt.ext.tornado.iostream.SSLIOStream( + self._stream = tornado.iostream.SSLIOStream( _sock, ssl_options={"cert_reqs": ssl.CERT_NONE} ) else: - self._stream = salt.ext.tornado.iostream.IOStream(_sock) + self._stream = tornado.iostream.IOStream(_sock) self._stream.set_close_callback(self.on_closed) self._stream.connect((self.host, self.port), self.on_connect) @@ -218,13 +219,11 @@ class IRCClient: event = self._event(raw) if event.code == "PING": - salt.ext.tornado.ioloop.IOLoop.current().spawn_callback( + tornado.ioloop.IOLoop.current().spawn_callback( self.send_message, "PONG {}".format(event.line) ) elif event.code == "PRIVMSG": - salt.ext.tornado.ioloop.IOLoop.current().spawn_callback( - self._privmsg, event - ) + tornado.ioloop.IOLoop.current().spawn_callback(self._privmsg, event) self.read_messages() def join_channel(self, channel): diff --git a/salt/engines/webhook.py b/salt/engines/webhook.py index 1424d4a27dc..ea7a60b481f 100644 --- a/salt/engines/webhook.py +++ b/salt/engines/webhook.py @@ -2,9 +2,10 @@ Send events from webhook api """ -import salt.ext.tornado.httpserver -import salt.ext.tornado.ioloop -import salt.ext.tornado.web +import tornado.httpserver +import tornado.ioloop +import tornado.web + import salt.utils.event @@ -63,9 +64,7 @@ def start(address=None, port=5000, ssl_crt=None, ssl_key=None): else: __salt__["event.send"](tag, msg) - class WebHook( - salt.ext.tornado.web.RequestHandler - ): # pylint: disable=abstract-method + class WebHook(tornado.web.RequestHandler): # pylint: disable=abstract-method def post(self, tag): # pylint: disable=arguments-differ body = self.request.body headers = self.request.headers @@ -75,14 +74,12 @@ def start(address=None, port=5000, ssl_crt=None, ssl_key=None): } fire("salt/engines/hook/" + tag, payload) - application = salt.ext.tornado.web.Application([(r"/(.*)", WebHook)]) + application = tornado.web.Application([(r"/(.*)", WebHook)]) ssl_options = None if all([ssl_crt, ssl_key]): ssl_options = {"certfile": ssl_crt, "keyfile": ssl_key} - io_loop = salt.ext.tornado.ioloop.IOLoop(make_current=False) + io_loop = tornado.ioloop.IOLoop(make_current=False) io_loop.make_current() - http_server = salt.ext.tornado.httpserver.HTTPServer( - application, ssl_options=ssl_options - ) + http_server = tornado.httpserver.HTTPServer(application, ssl_options=ssl_options) http_server.listen(port, address=address) io_loop.start() diff --git a/salt/ext/tornado/__init__.py b/salt/ext/tornado/__init__.py deleted file mode 100644 index 3046f024801..00000000000 --- a/salt/ext/tornado/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""The Tornado web server and tools.""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -# version is a human-readable version number. - -# version_info is a four-tuple for programmatic comparison. The first -# three numbers are the components of the version number. The fourth -# is zero for an official release, positive for a development branch, -# or negative for a release candidate or beta (after the base version -# number has been incremented) -version = "4.5.3" -version_info = (4, 5, 3, 0) diff --git a/salt/ext/tornado/_locale_data.py b/salt/ext/tornado/_locale_data.py deleted file mode 100644 index 4f77fb1209c..00000000000 --- a/salt/ext/tornado/_locale_data.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 -# -# Copyright 2012 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Data used by the tornado.locale module.""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -LOCALE_NAMES = { - "af_ZA": {"name_en": u"Afrikaans", "name": u"Afrikaans"}, - "am_ET": {"name_en": u"Amharic", "name": u"አማርኛ"}, - "ar_AR": {"name_en": u"Arabic", "name": u"العربية"}, - "bg_BG": {"name_en": u"Bulgarian", "name": u"Български"}, - "bn_IN": {"name_en": u"Bengali", "name": u"বাংলা"}, - "bs_BA": {"name_en": u"Bosnian", "name": u"Bosanski"}, - "ca_ES": {"name_en": u"Catalan", "name": u"Català"}, - "cs_CZ": {"name_en": u"Czech", "name": u"Čeština"}, - "cy_GB": {"name_en": u"Welsh", "name": u"Cymraeg"}, - "da_DK": {"name_en": u"Danish", "name": u"Dansk"}, - "de_DE": {"name_en": u"German", "name": u"Deutsch"}, - "el_GR": {"name_en": u"Greek", "name": u"Ελληνικά"}, - "en_GB": {"name_en": u"English (UK)", "name": u"English (UK)"}, - "en_US": {"name_en": u"English (US)", "name": u"English (US)"}, - "es_ES": {"name_en": u"Spanish (Spain)", "name": u"Español (España)"}, - "es_LA": {"name_en": u"Spanish", "name": u"Español"}, - "et_EE": {"name_en": u"Estonian", "name": u"Eesti"}, - "eu_ES": {"name_en": u"Basque", "name": u"Euskara"}, - "fa_IR": {"name_en": u"Persian", "name": u"فارسی"}, - "fi_FI": {"name_en": u"Finnish", "name": u"Suomi"}, - "fr_CA": {"name_en": u"French (Canada)", "name": u"Français (Canada)"}, - "fr_FR": {"name_en": u"French", "name": u"Français"}, - "ga_IE": {"name_en": u"Irish", "name": u"Gaeilge"}, - "gl_ES": {"name_en": u"Galician", "name": u"Galego"}, - "he_IL": {"name_en": u"Hebrew", "name": u"עברית"}, - "hi_IN": {"name_en": u"Hindi", "name": u"हिन्दी"}, - "hr_HR": {"name_en": u"Croatian", "name": u"Hrvatski"}, - "hu_HU": {"name_en": u"Hungarian", "name": u"Magyar"}, - "id_ID": {"name_en": u"Indonesian", "name": u"Bahasa Indonesia"}, - "is_IS": {"name_en": u"Icelandic", "name": u"Íslenska"}, - "it_IT": {"name_en": u"Italian", "name": u"Italiano"}, - "ja_JP": {"name_en": u"Japanese", "name": u"日本語"}, - "ko_KR": {"name_en": u"Korean", "name": u"한국어"}, - "lt_LT": {"name_en": u"Lithuanian", "name": u"Lietuvių"}, - "lv_LV": {"name_en": u"Latvian", "name": u"Latviešu"}, - "mk_MK": {"name_en": u"Macedonian", "name": u"Македонски"}, - "ml_IN": {"name_en": u"Malayalam", "name": u"മലയാളം"}, - "ms_MY": {"name_en": u"Malay", "name": u"Bahasa Melayu"}, - "nb_NO": {"name_en": u"Norwegian (bokmal)", "name": u"Norsk (bokmål)"}, - "nl_NL": {"name_en": u"Dutch", "name": u"Nederlands"}, - "nn_NO": {"name_en": u"Norwegian (nynorsk)", "name": u"Norsk (nynorsk)"}, - "pa_IN": {"name_en": u"Punjabi", "name": u"ਪੰਜਾਬੀ"}, - "pl_PL": {"name_en": u"Polish", "name": u"Polski"}, - "pt_BR": {"name_en": u"Portuguese (Brazil)", "name": u"Português (Brasil)"}, - "pt_PT": {"name_en": u"Portuguese (Portugal)", "name": u"Português (Portugal)"}, - "ro_RO": {"name_en": u"Romanian", "name": u"Română"}, - "ru_RU": {"name_en": u"Russian", "name": u"Русский"}, - "sk_SK": {"name_en": u"Slovak", "name": u"Slovenčina"}, - "sl_SI": {"name_en": u"Slovenian", "name": u"Slovenščina"}, - "sq_AL": {"name_en": u"Albanian", "name": u"Shqip"}, - "sr_RS": {"name_en": u"Serbian", "name": u"Српски"}, - "sv_SE": {"name_en": u"Swedish", "name": u"Svenska"}, - "sw_KE": {"name_en": u"Swahili", "name": u"Kiswahili"}, - "ta_IN": {"name_en": u"Tamil", "name": u"தமிழ்"}, - "te_IN": {"name_en": u"Telugu", "name": u"తెలుగు"}, - "th_TH": {"name_en": u"Thai", "name": u"ภาษาไทย"}, - "tl_PH": {"name_en": u"Filipino", "name": u"Filipino"}, - "tr_TR": {"name_en": u"Turkish", "name": u"Türkçe"}, - "uk_UA": {"name_en": u"Ukraini ", "name": u"Українська"}, - "vi_VN": {"name_en": u"Vietnamese", "name": u"Tiếng Việt"}, - "zh_CN": {"name_en": u"Chinese (Simplified)", "name": u"中文(简体)"}, - "zh_TW": {"name_en": u"Chinese (Traditional)", "name": u"中文(繁體)"}, -} diff --git a/salt/ext/tornado/auth.py b/salt/ext/tornado/auth.py deleted file mode 100644 index d43fc1273d2..00000000000 --- a/salt/ext/tornado/auth.py +++ /dev/null @@ -1,1155 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""This module contains implementations of various third-party -authentication schemes. - -All the classes in this file are class mixins designed to be used with -the `tornado.web.RequestHandler` class. They are used in two ways: - -* On a login handler, use methods such as ``authenticate_redirect()``, - ``authorize_redirect()``, and ``get_authenticated_user()`` to - establish the user's identity and store authentication tokens to your - database and/or cookies. -* In non-login handlers, use methods such as ``facebook_request()`` - or ``twitter_request()`` to use the authentication tokens to make - requests to the respective services. - -They all take slightly different arguments due to the fact all these -services implement authentication and authorization slightly differently. -See the individual service classes below for complete documentation. - -Example usage for Google OAuth: - -.. testcode:: - - class GoogleOAuth2LoginHandler(tornado.web.RequestHandler, - tornado.auth.GoogleOAuth2Mixin): - @tornado.gen.coroutine - def get(self): - if self.get_argument('code', False): - user = yield self.get_authenticated_user( - redirect_uri='http://your.site.com/auth/google', - code=self.get_argument('code')) - # Save the user with e.g. set_secure_cookie - else: - yield self.authorize_redirect( - redirect_uri='http://your.site.com/auth/google', - client_id=self.settings['google_oauth']['key'], - scope=['profile', 'email'], - response_type='code', - extra_params={'approval_prompt': 'auto'}) - -.. testoutput:: - :hide: - - -.. versionchanged:: 4.0 - All of the callback interfaces in this module are now guaranteed - to run their callback with an argument of ``None`` on error. - Previously some functions would do this while others would simply - terminate the request on their own. This change also ensures that - errors are more consistently reported through the ``Future`` interfaces. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import base64 -import binascii -import functools -import hashlib -import hmac -import time -import uuid - -from salt.ext.tornado.concurrent import TracebackFuture, return_future, chain_future -from salt.ext.tornado import gen -from salt.ext.tornado import httpclient -from salt.ext.tornado import escape -from salt.ext.tornado.httputil import url_concat -from salt.ext.tornado.log import gen_log -from salt.ext.tornado.stack_context import ExceptionStackContext -from salt.ext.tornado.util import unicode_type, ArgReplacer, PY3 - -if PY3: - import urllib.parse as urlparse - import urllib.parse as urllib_parse - long = int -else: - import urlparse - import urllib as urllib_parse - - -class AuthError(Exception): - pass - - -def _auth_future_to_callback(callback, future): - try: - result = future.result() - except AuthError as e: - gen_log.warning(str(e)) - result = None - callback(result) - - -def _auth_return_future(f): - """Similar to tornado.concurrent.return_future, but uses the auth - module's legacy callback interface. - - Note that when using this decorator the ``callback`` parameter - inside the function will actually be a future. - """ - replacer = ArgReplacer(f, 'callback') - - @functools.wraps(f) - def wrapper(*args, **kwargs): - future = TracebackFuture() - callback, args, kwargs = replacer.replace(future, args, kwargs) - if callback is not None: - future.add_done_callback( - functools.partial(_auth_future_to_callback, callback)) - - def handle_exception(typ, value, tb): - if future.done(): - return False - else: - future.set_exc_info((typ, value, tb)) - return True - with ExceptionStackContext(handle_exception): - f(*args, **kwargs) - return future - return wrapper - - -class OpenIdMixin(object): - """Abstract implementation of OpenID and Attribute Exchange. - - Class attributes: - - * ``_OPENID_ENDPOINT``: the identity provider's URI. - """ - @return_future - def authenticate_redirect(self, callback_uri=None, - ax_attrs=["name", "email", "language", "username"], - callback=None): - """Redirects to the authentication URL for this service. - - After authentication, the service will redirect back to the given - callback URI with additional parameters including ``openid.mode``. - - We request the given attributes for the authenticated user by - default (name, email, language, and username). If you don't need - all those attributes for your app, you can request fewer with - the ax_attrs keyword argument. - - .. versionchanged:: 3.1 - Returns a `.Future` and takes an optional callback. These are - not strictly necessary as this method is synchronous, - but they are supplied for consistency with - `OAuthMixin.authorize_redirect`. - """ - callback_uri = callback_uri or self.request.uri - args = self._openid_args(callback_uri, ax_attrs=ax_attrs) - self.redirect(self._OPENID_ENDPOINT + "?" + urllib_parse.urlencode(args)) - callback() - - @_auth_return_future - def get_authenticated_user(self, callback, http_client=None): - """Fetches the authenticated user data upon redirect. - - This method should be called by the handler that receives the - redirect from the `authenticate_redirect()` method (which is - often the same as the one that calls it; in that case you would - call `get_authenticated_user` if the ``openid.mode`` parameter - is present and `authenticate_redirect` if it is not). - - The result of this method will generally be used to set a cookie. - """ - # Verify the OpenID response via direct request to the OP - args = dict((k, v[-1]) for k, v in self.request.arguments.items()) - args["openid.mode"] = u"check_authentication" - url = self._OPENID_ENDPOINT - if http_client is None: - http_client = self.get_auth_http_client() - http_client.fetch(url, functools.partial( - self._on_authentication_verified, callback), - method="POST", body=urllib_parse.urlencode(args)) - - def _openid_args(self, callback_uri, ax_attrs=[], oauth_scope=None): - url = urlparse.urljoin(self.request.full_url(), callback_uri) - args = { - "openid.ns": "http://specs.openid.net/auth/2.0", - "openid.claimed_id": - "http://specs.openid.net/auth/2.0/identifier_select", - "openid.identity": - "http://specs.openid.net/auth/2.0/identifier_select", - "openid.return_to": url, - "openid.realm": urlparse.urljoin(url, '/'), - "openid.mode": "checkid_setup", - } - if ax_attrs: - args.update({ - "openid.ns.ax": "http://openid.net/srv/ax/1.0", - "openid.ax.mode": "fetch_request", - }) - ax_attrs = set(ax_attrs) - required = [] - if "name" in ax_attrs: - ax_attrs -= set(["name", "firstname", "fullname", "lastname"]) - required += ["firstname", "fullname", "lastname"] - args.update({ - "openid.ax.type.firstname": - "http://axschema.org/namePerson/first", - "openid.ax.type.fullname": - "http://axschema.org/namePerson", - "openid.ax.type.lastname": - "http://axschema.org/namePerson/last", - }) - known_attrs = { - "email": "http://axschema.org/contact/email", - "language": "http://axschema.org/pref/language", - "username": "http://axschema.org/namePerson/friendly", - } - for name in ax_attrs: - args["openid.ax.type." + name] = known_attrs[name] - required.append(name) - args["openid.ax.required"] = ",".join(required) - if oauth_scope: - args.update({ - "openid.ns.oauth": - "http://specs.openid.net/extensions/oauth/1.0", - "openid.oauth.consumer": self.request.host.split(":")[0], - "openid.oauth.scope": oauth_scope, - }) - return args - - def _on_authentication_verified(self, future, response): - if response.error or b"is_valid:true" not in response.body: - future.set_exception(AuthError( - "Invalid OpenID response: %s" % (response.error or - response.body))) - return - - # Make sure we got back at least an email from attribute exchange - ax_ns = None - for name in self.request.arguments: - if name.startswith("openid.ns.") and \ - self.get_argument(name) == u"http://openid.net/srv/ax/1.0": - ax_ns = name[10:] - break - - def get_ax_arg(uri): - if not ax_ns: - return u"" - prefix = "openid." + ax_ns + ".type." - ax_name = None - for name in self.request.arguments.keys(): - if self.get_argument(name) == uri and name.startswith(prefix): - part = name[len(prefix):] - ax_name = "openid." + ax_ns + ".value." + part - break - if not ax_name: - return u"" - return self.get_argument(ax_name, u"") - - email = get_ax_arg("http://axschema.org/contact/email") - name = get_ax_arg("http://axschema.org/namePerson") - first_name = get_ax_arg("http://axschema.org/namePerson/first") - last_name = get_ax_arg("http://axschema.org/namePerson/last") - username = get_ax_arg("http://axschema.org/namePerson/friendly") - locale = get_ax_arg("http://axschema.org/pref/language").lower() - user = dict() - name_parts = [] - if first_name: - user["first_name"] = first_name - name_parts.append(first_name) - if last_name: - user["last_name"] = last_name - name_parts.append(last_name) - if name: - user["name"] = name - elif name_parts: - user["name"] = u" ".join(name_parts) - elif email: - user["name"] = email.split("@")[0] - if email: - user["email"] = email - if locale: - user["locale"] = locale - if username: - user["username"] = username - claimed_id = self.get_argument("openid.claimed_id", None) - if claimed_id: - user["claimed_id"] = claimed_id - future.set_result(user) - - def get_auth_http_client(self): - """Returns the `.AsyncHTTPClient` instance to be used for auth requests. - - May be overridden by subclasses to use an HTTP client other than - the default. - """ - return httpclient.AsyncHTTPClient() - - -class OAuthMixin(object): - """Abstract implementation of OAuth 1.0 and 1.0a. - - See `TwitterMixin` below for an example implementation. - - Class attributes: - - * ``_OAUTH_AUTHORIZE_URL``: The service's OAuth authorization url. - * ``_OAUTH_ACCESS_TOKEN_URL``: The service's OAuth access token url. - * ``_OAUTH_VERSION``: May be either "1.0" or "1.0a". - * ``_OAUTH_NO_CALLBACKS``: Set this to True if the service requires - advance registration of callbacks. - - Subclasses must also override the `_oauth_get_user_future` and - `_oauth_consumer_token` methods. - """ - @return_future - def authorize_redirect(self, callback_uri=None, extra_params=None, - http_client=None, callback=None): - """Redirects the user to obtain OAuth authorization for this service. - - The ``callback_uri`` may be omitted if you have previously - registered a callback URI with the third-party service. For - some services (including Friendfeed), you must use a - previously-registered callback URI and cannot specify a - callback via this method. - - This method sets a cookie called ``_oauth_request_token`` which is - subsequently used (and cleared) in `get_authenticated_user` for - security purposes. - - Note that this method is asynchronous, although it calls - `.RequestHandler.finish` for you so it may not be necessary - to pass a callback or use the `.Future` it returns. However, - if this method is called from a function decorated with - `.gen.coroutine`, you must call it with ``yield`` to keep the - response from being closed prematurely. - - .. versionchanged:: 3.1 - Now returns a `.Future` and takes an optional callback, for - compatibility with `.gen.coroutine`. - """ - if callback_uri and getattr(self, "_OAUTH_NO_CALLBACKS", False): - raise Exception("This service does not support oauth_callback") - if http_client is None: - http_client = self.get_auth_http_client() - if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - http_client.fetch( - self._oauth_request_token_url(callback_uri=callback_uri, - extra_params=extra_params), - functools.partial( - self._on_request_token, - self._OAUTH_AUTHORIZE_URL, - callback_uri, - callback)) - else: - http_client.fetch( - self._oauth_request_token_url(), - functools.partial( - self._on_request_token, self._OAUTH_AUTHORIZE_URL, - callback_uri, - callback)) - - @_auth_return_future - def get_authenticated_user(self, callback, http_client=None): - """Gets the OAuth authorized user and access token. - - This method should be called from the handler for your - OAuth callback URL to complete the registration process. We run the - callback with the authenticated user dictionary. This dictionary - will contain an ``access_key`` which can be used to make authorized - requests to this service on behalf of the user. The dictionary will - also contain other fields such as ``name``, depending on the service - used. - """ - future = callback - request_key = escape.utf8(self.get_argument("oauth_token")) - oauth_verifier = self.get_argument("oauth_verifier", None) - request_cookie = self.get_cookie("_oauth_request_token") - if not request_cookie: - future.set_exception(AuthError( - "Missing OAuth request token cookie")) - return - self.clear_cookie("_oauth_request_token") - cookie_key, cookie_secret = [base64.b64decode(escape.utf8(i)) for i in request_cookie.split("|")] - if cookie_key != request_key: - future.set_exception(AuthError( - "Request token does not match cookie")) - return - token = dict(key=cookie_key, secret=cookie_secret) - if oauth_verifier: - token["verifier"] = oauth_verifier - if http_client is None: - http_client = self.get_auth_http_client() - http_client.fetch(self._oauth_access_token_url(token), - functools.partial(self._on_access_token, callback)) - - def _oauth_request_token_url(self, callback_uri=None, extra_params=None): - consumer_token = self._oauth_consumer_token() - url = self._OAUTH_REQUEST_TOKEN_URL - args = dict( - oauth_consumer_key=escape.to_basestring(consumer_token["key"]), - oauth_signature_method="HMAC-SHA1", - oauth_timestamp=str(int(time.time())), - oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)), - oauth_version="1.0", - ) - if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - if callback_uri == "oob": - args["oauth_callback"] = "oob" - elif callback_uri: - args["oauth_callback"] = urlparse.urljoin( - self.request.full_url(), callback_uri) - if extra_params: - args.update(extra_params) - signature = _oauth10a_signature(consumer_token, "GET", url, args) - else: - signature = _oauth_signature(consumer_token, "GET", url, args) - - args["oauth_signature"] = signature - return url + "?" + urllib_parse.urlencode(args) - - def _on_request_token(self, authorize_url, callback_uri, callback, - response): - if response.error: - raise Exception("Could not get request token: %s" % response.error) - request_token = _oauth_parse_response(response.body) - data = (base64.b64encode(escape.utf8(request_token["key"])) + b"|" + - base64.b64encode(escape.utf8(request_token["secret"]))) - self.set_cookie("_oauth_request_token", data) - args = dict(oauth_token=request_token["key"]) - if callback_uri == "oob": - self.finish(authorize_url + "?" + urllib_parse.urlencode(args)) - callback() - return - elif callback_uri: - args["oauth_callback"] = urlparse.urljoin( - self.request.full_url(), callback_uri) - self.redirect(authorize_url + "?" + urllib_parse.urlencode(args)) - callback() - - def _oauth_access_token_url(self, request_token): - consumer_token = self._oauth_consumer_token() - url = self._OAUTH_ACCESS_TOKEN_URL - args = dict( - oauth_consumer_key=escape.to_basestring(consumer_token["key"]), - oauth_token=escape.to_basestring(request_token["key"]), - oauth_signature_method="HMAC-SHA1", - oauth_timestamp=str(int(time.time())), - oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)), - oauth_version="1.0", - ) - if "verifier" in request_token: - args["oauth_verifier"] = request_token["verifier"] - - if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - signature = _oauth10a_signature(consumer_token, "GET", url, args, - request_token) - else: - signature = _oauth_signature(consumer_token, "GET", url, args, - request_token) - - args["oauth_signature"] = signature - return url + "?" + urllib_parse.urlencode(args) - - def _on_access_token(self, future, response): - if response.error: - future.set_exception(AuthError("Could not fetch access token")) - return - - access_token = _oauth_parse_response(response.body) - self._oauth_get_user_future(access_token).add_done_callback( - functools.partial(self._on_oauth_get_user, access_token, future)) - - def _oauth_consumer_token(self): - """Subclasses must override this to return their OAuth consumer keys. - - The return value should be a `dict` with keys ``key`` and ``secret``. - """ - raise NotImplementedError() - - @return_future - def _oauth_get_user_future(self, access_token, callback): - """Subclasses must override this to get basic information about the - user. - - Should return a `.Future` whose result is a dictionary - containing information about the user, which may have been - retrieved by using ``access_token`` to make a request to the - service. - - The access token will be added to the returned dictionary to make - the result of `get_authenticated_user`. - - For backwards compatibility, the callback-based ``_oauth_get_user`` - method is also supported. - """ - # By default, call the old-style _oauth_get_user, but new code - # should override this method instead. - self._oauth_get_user(access_token, callback) - - def _oauth_get_user(self, access_token, callback): - raise NotImplementedError() - - def _on_oauth_get_user(self, access_token, future, user_future): - if user_future.exception() is not None: - future.set_exception(user_future.exception()) - return - user = user_future.result() - if not user: - future.set_exception(AuthError("Error getting user")) - return - user["access_token"] = access_token - future.set_result(user) - - def _oauth_request_parameters(self, url, access_token, parameters={}, - method="GET"): - """Returns the OAuth parameters as a dict for the given request. - - parameters should include all POST arguments and query string arguments - that will be sent with the request. - """ - consumer_token = self._oauth_consumer_token() - base_args = dict( - oauth_consumer_key=escape.to_basestring(consumer_token["key"]), - oauth_token=escape.to_basestring(access_token["key"]), - oauth_signature_method="HMAC-SHA1", - oauth_timestamp=str(int(time.time())), - oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)), - oauth_version="1.0", - ) - args = {} - args.update(base_args) - args.update(parameters) - if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - signature = _oauth10a_signature(consumer_token, method, url, args, - access_token) - else: - signature = _oauth_signature(consumer_token, method, url, args, - access_token) - base_args["oauth_signature"] = escape.to_basestring(signature) - return base_args - - def get_auth_http_client(self): - """Returns the `.AsyncHTTPClient` instance to be used for auth requests. - - May be overridden by subclasses to use an HTTP client other than - the default. - """ - return httpclient.AsyncHTTPClient() - - -class OAuth2Mixin(object): - """Abstract implementation of OAuth 2.0. - - See `FacebookGraphMixin` or `GoogleOAuth2Mixin` below for example - implementations. - - Class attributes: - - * ``_OAUTH_AUTHORIZE_URL``: The service's authorization url. - * ``_OAUTH_ACCESS_TOKEN_URL``: The service's access token url. - """ - @return_future - def authorize_redirect(self, redirect_uri=None, client_id=None, - client_secret=None, extra_params=None, - callback=None, scope=None, response_type="code"): - """Redirects the user to obtain OAuth authorization for this service. - - Some providers require that you register a redirect URL with - your application instead of passing one via this method. You - should call this method to log the user in, and then call - ``get_authenticated_user`` in the handler for your - redirect URL to complete the authorization process. - - .. versionchanged:: 3.1 - Returns a `.Future` and takes an optional callback. These are - not strictly necessary as this method is synchronous, - but they are supplied for consistency with - `OAuthMixin.authorize_redirect`. - """ - args = { - "redirect_uri": redirect_uri, - "client_id": client_id, - "response_type": response_type - } - if extra_params: - args.update(extra_params) - if scope: - args['scope'] = ' '.join(scope) - self.redirect( - url_concat(self._OAUTH_AUTHORIZE_URL, args)) - callback() - - def _oauth_request_token_url(self, redirect_uri=None, client_id=None, - client_secret=None, code=None, - extra_params=None): - url = self._OAUTH_ACCESS_TOKEN_URL - args = dict( - redirect_uri=redirect_uri, - code=code, - client_id=client_id, - client_secret=client_secret, - ) - if extra_params: - args.update(extra_params) - return url_concat(url, args) - - @_auth_return_future - def oauth2_request(self, url, callback, access_token=None, - post_args=None, **args): - """Fetches the given URL auth an OAuth2 access token. - - If the request is a POST, ``post_args`` should be provided. Query - string arguments should be given as keyword arguments. - - Example usage: - - ..testcode:: - - class MainHandler(tornado.web.RequestHandler, - tornado.auth.FacebookGraphMixin): - @tornado.web.authenticated - @tornado.gen.coroutine - def get(self): - new_entry = yield self.oauth2_request( - "https://graph.facebook.com/me/feed", - post_args={"message": "I am posting from my Tornado application!"}, - access_token=self.current_user["access_token"]) - - if not new_entry: - # Call failed; perhaps missing permission? - yield self.authorize_redirect() - return - self.finish("Posted a message!") - - .. testoutput:: - :hide: - - .. versionadded:: 4.3 - """ - all_args = {} - if access_token: - all_args["access_token"] = access_token - all_args.update(args) - - if all_args: - url += "?" + urllib_parse.urlencode(all_args) - callback = functools.partial(self._on_oauth2_request, callback) - http = self.get_auth_http_client() - if post_args is not None: - http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), - callback=callback) - else: - http.fetch(url, callback=callback) - - def _on_oauth2_request(self, future, response): - if response.error: - future.set_exception(AuthError("Error response %s fetching %s" % - (response.error, response.request.url))) - return - - future.set_result(escape.json_decode(response.body)) - - def get_auth_http_client(self): - """Returns the `.AsyncHTTPClient` instance to be used for auth requests. - - May be overridden by subclasses to use an HTTP client other than - the default. - - .. versionadded:: 4.3 - """ - return httpclient.AsyncHTTPClient() - - -class TwitterMixin(OAuthMixin): - """Twitter OAuth authentication. - - To authenticate with Twitter, register your application with - Twitter at http://twitter.com/apps. Then copy your Consumer Key - and Consumer Secret to the application - `~tornado.web.Application.settings` ``twitter_consumer_key`` and - ``twitter_consumer_secret``. Use this mixin on the handler for the - URL you registered as your application's callback URL. - - When your application is set up, you can use this mixin like this - to authenticate the user with Twitter and get access to their stream: - - .. testcode:: - - class TwitterLoginHandler(tornado.web.RequestHandler, - tornado.auth.TwitterMixin): - @tornado.gen.coroutine - def get(self): - if self.get_argument("oauth_token", None): - user = yield self.get_authenticated_user() - # Save the user using e.g. set_secure_cookie() - else: - yield self.authorize_redirect() - - .. testoutput:: - :hide: - - The user object returned by `~OAuthMixin.get_authenticated_user` - includes the attributes ``username``, ``name``, ``access_token``, - and all of the custom Twitter user attributes described at - https://dev.twitter.com/docs/api/1.1/get/users/show - """ - _OAUTH_REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token" - _OAUTH_ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token" - _OAUTH_AUTHORIZE_URL = "https://api.twitter.com/oauth/authorize" - _OAUTH_AUTHENTICATE_URL = "https://api.twitter.com/oauth/authenticate" - _OAUTH_NO_CALLBACKS = False - _TWITTER_BASE_URL = "https://api.twitter.com/1.1" - - @return_future - def authenticate_redirect(self, callback_uri=None, callback=None): - """Just like `~OAuthMixin.authorize_redirect`, but - auto-redirects if authorized. - - This is generally the right interface to use if you are using - Twitter for single-sign on. - - .. versionchanged:: 3.1 - Now returns a `.Future` and takes an optional callback, for - compatibility with `.gen.coroutine`. - """ - http = self.get_auth_http_client() - http.fetch(self._oauth_request_token_url(callback_uri=callback_uri), - functools.partial( - self._on_request_token, self._OAUTH_AUTHENTICATE_URL, - None, callback)) - - @_auth_return_future - def twitter_request(self, path, callback=None, access_token=None, - post_args=None, **args): - """Fetches the given API path, e.g., ``statuses/user_timeline/btaylor`` - - The path should not include the format or API version number. - (we automatically use JSON format and API version 1). - - If the request is a POST, ``post_args`` should be provided. Query - string arguments should be given as keyword arguments. - - All the Twitter methods are documented at http://dev.twitter.com/ - - Many methods require an OAuth access token which you can - obtain through `~OAuthMixin.authorize_redirect` and - `~OAuthMixin.get_authenticated_user`. The user returned through that - process includes an 'access_token' attribute that can be used - to make authenticated requests via this method. Example - usage: - - .. testcode:: - - class MainHandler(tornado.web.RequestHandler, - tornado.auth.TwitterMixin): - @tornado.web.authenticated - @tornado.gen.coroutine - def get(self): - new_entry = yield self.twitter_request( - "/statuses/update", - post_args={"status": "Testing Tornado Web Server"}, - access_token=self.current_user["access_token"]) - if not new_entry: - # Call failed; perhaps missing permission? - yield self.authorize_redirect() - return - self.finish("Posted a message!") - - .. testoutput:: - :hide: - - """ - if path.startswith('http:') or path.startswith('https:'): - # Raw urls are useful for e.g. search which doesn't follow the - # usual pattern: http://search.twitter.com/search.json - url = path - else: - url = self._TWITTER_BASE_URL + path + ".json" - # Add the OAuth resource request signature if we have credentials - if access_token: - all_args = {} - all_args.update(args) - all_args.update(post_args or {}) - method = "POST" if post_args is not None else "GET" - oauth = self._oauth_request_parameters( - url, access_token, all_args, method=method) - args.update(oauth) - if args: - url += "?" + urllib_parse.urlencode(args) - http = self.get_auth_http_client() - http_callback = functools.partial(self._on_twitter_request, callback) - if post_args is not None: - http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), - callback=http_callback) - else: - http.fetch(url, callback=http_callback) - - def _on_twitter_request(self, future, response): - if response.error: - future.set_exception(AuthError( - "Error response %s fetching %s" % (response.error, - response.request.url))) - return - future.set_result(escape.json_decode(response.body)) - - def _oauth_consumer_token(self): - self.require_setting("twitter_consumer_key", "Twitter OAuth") - self.require_setting("twitter_consumer_secret", "Twitter OAuth") - return dict( - key=self.settings["twitter_consumer_key"], - secret=self.settings["twitter_consumer_secret"]) - - @gen.coroutine - def _oauth_get_user_future(self, access_token): - user = yield self.twitter_request( - "/account/verify_credentials", - access_token=access_token) - if user: - user["username"] = user["screen_name"] - raise gen.Return(user) - - -class GoogleOAuth2Mixin(OAuth2Mixin): - """Google authentication using OAuth2. - - In order to use, register your application with Google and copy the - relevant parameters to your application settings. - - * Go to the Google Dev Console at http://console.developers.google.com - * Select a project, or create a new one. - * In the sidebar on the left, select APIs & Auth. - * In the list of APIs, find the Google+ API service and set it to ON. - * In the sidebar on the left, select Credentials. - * In the OAuth section of the page, select Create New Client ID. - * Set the Redirect URI to point to your auth handler - * Copy the "Client secret" and "Client ID" to the application settings as - {"google_oauth": {"key": CLIENT_ID, "secret": CLIENT_SECRET}} - - .. versionadded:: 3.2 - """ - _OAUTH_AUTHORIZE_URL = "https://accounts.google.com/o/oauth2/auth" - _OAUTH_ACCESS_TOKEN_URL = "https://accounts.google.com/o/oauth2/token" - _OAUTH_USERINFO_URL = "https://www.googleapis.com/oauth2/v1/userinfo" - _OAUTH_NO_CALLBACKS = False - _OAUTH_SETTINGS_KEY = 'google_oauth' - - @_auth_return_future - def get_authenticated_user(self, redirect_uri, code, callback): - """Handles the login for the Google user, returning an access token. - - The result is a dictionary containing an ``access_token`` field - ([among others](https://developers.google.com/identity/protocols/OAuth2WebServer#handlingtheresponse)). - Unlike other ``get_authenticated_user`` methods in this package, - this method does not return any additional information about the user. - The returned access token can be used with `OAuth2Mixin.oauth2_request` - to request additional information (perhaps from - ``https://www.googleapis.com/oauth2/v2/userinfo``) - - Example usage: - - .. testcode:: - - class GoogleOAuth2LoginHandler(tornado.web.RequestHandler, - tornado.auth.GoogleOAuth2Mixin): - @tornado.gen.coroutine - def get(self): - if self.get_argument('code', False): - access = yield self.get_authenticated_user( - redirect_uri='http://your.site.com/auth/google', - code=self.get_argument('code')) - user = yield self.oauth2_request( - "https://www.googleapis.com/oauth2/v1/userinfo", - access_token=access["access_token"]) - # Save the user and access token with - # e.g. set_secure_cookie. - else: - yield self.authorize_redirect( - redirect_uri='http://your.site.com/auth/google', - client_id=self.settings['google_oauth']['key'], - scope=['profile', 'email'], - response_type='code', - extra_params={'approval_prompt': 'auto'}) - - .. testoutput:: - :hide: - - """ - http = self.get_auth_http_client() - body = urllib_parse.urlencode({ - "redirect_uri": redirect_uri, - "code": code, - "client_id": self.settings[self._OAUTH_SETTINGS_KEY]['key'], - "client_secret": self.settings[self._OAUTH_SETTINGS_KEY]['secret'], - "grant_type": "authorization_code", - }) - - http.fetch(self._OAUTH_ACCESS_TOKEN_URL, - functools.partial(self._on_access_token, callback), - method="POST", headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=body) - - def _on_access_token(self, future, response): - """Callback function for the exchange to the access token.""" - if response.error: - future.set_exception(AuthError('Google auth error: %s' % str(response))) - return - - args = escape.json_decode(response.body) - future.set_result(args) - - -class FacebookGraphMixin(OAuth2Mixin): - """Facebook authentication using the new Graph API and OAuth2.""" - _OAUTH_ACCESS_TOKEN_URL = "https://graph.facebook.com/oauth/access_token?" - _OAUTH_AUTHORIZE_URL = "https://www.facebook.com/dialog/oauth?" - _OAUTH_NO_CALLBACKS = False - _FACEBOOK_BASE_URL = "https://graph.facebook.com" - - @_auth_return_future - def get_authenticated_user(self, redirect_uri, client_id, client_secret, - code, callback, extra_fields=None): - """Handles the login for the Facebook user, returning a user object. - - Example usage: - - .. testcode:: - - class FacebookGraphLoginHandler(tornado.web.RequestHandler, - tornado.auth.FacebookGraphMixin): - @tornado.gen.coroutine - def get(self): - if self.get_argument("code", False): - user = yield self.get_authenticated_user( - redirect_uri='/auth/facebookgraph/', - client_id=self.settings["facebook_api_key"], - client_secret=self.settings["facebook_secret"], - code=self.get_argument("code")) - # Save the user with e.g. set_secure_cookie - else: - yield self.authorize_redirect( - redirect_uri='/auth/facebookgraph/', - client_id=self.settings["facebook_api_key"], - extra_params={"scope": "read_stream,offline_access"}) - - .. testoutput:: - :hide: - - This method returns a dictionary which may contain the following fields: - - * ``access_token``, a string which may be passed to `facebook_request` - * ``session_expires``, an integer encoded as a string representing - the time until the access token expires in seconds. This field should - be used like ``int(user['session_expires'])``; in a future version of - Tornado it will change from a string to an integer. - * ``id``, ``name``, ``first_name``, ``last_name``, ``locale``, ``picture``, - ``link``, plus any fields named in the ``extra_fields`` argument. These - fields are copied from the Facebook graph API `user object `_ - - .. versionchanged:: 4.5 - The ``session_expires`` field was updated to support changes made to the - Facebook API in March 2017. - """ - http = self.get_auth_http_client() - args = { - "redirect_uri": redirect_uri, - "code": code, - "client_id": client_id, - "client_secret": client_secret, - } - - fields = set(['id', 'name', 'first_name', 'last_name', - 'locale', 'picture', 'link']) - if extra_fields: - fields.update(extra_fields) - - http.fetch(self._oauth_request_token_url(**args), - functools.partial(self._on_access_token, redirect_uri, client_id, - client_secret, callback, fields)) - - def _on_access_token(self, redirect_uri, client_id, client_secret, - future, fields, response): - if response.error: - future.set_exception(AuthError('Facebook auth error: %s' % str(response))) - return - - args = escape.json_decode(response.body) - session = { - "access_token": args.get("access_token"), - "expires_in": args.get("expires_in") - } - - self.facebook_request( - path="/me", - callback=functools.partial( - self._on_get_user_info, future, session, fields), - access_token=session["access_token"], - appsecret_proof=hmac.new(key=client_secret.encode('utf8'), - msg=session["access_token"].encode('utf8'), - digestmod=hashlib.sha256).hexdigest(), - fields=",".join(fields) - ) - - def _on_get_user_info(self, future, session, fields, user): - if user is None: - future.set_result(None) - return - - fieldmap = {} - for field in fields: - fieldmap[field] = user.get(field) - - # session_expires is converted to str for compatibility with - # older versions in which the server used url-encoding and - # this code simply returned the string verbatim. - # This should change in Tornado 5.0. - fieldmap.update({"access_token": session["access_token"], - "session_expires": str(session.get("expires_in"))}) - future.set_result(fieldmap) - - @_auth_return_future - def facebook_request(self, path, callback, access_token=None, - post_args=None, **args): - """Fetches the given relative API path, e.g., "/btaylor/picture" - - If the request is a POST, ``post_args`` should be provided. Query - string arguments should be given as keyword arguments. - - An introduction to the Facebook Graph API can be found at - http://developers.facebook.com/docs/api - - Many methods require an OAuth access token which you can - obtain through `~OAuth2Mixin.authorize_redirect` and - `get_authenticated_user`. The user returned through that - process includes an ``access_token`` attribute that can be - used to make authenticated requests via this method. - - Example usage: - - ..testcode:: - - class MainHandler(tornado.web.RequestHandler, - tornado.auth.FacebookGraphMixin): - @tornado.web.authenticated - @tornado.gen.coroutine - def get(self): - new_entry = yield self.facebook_request( - "/me/feed", - post_args={"message": "I am posting from my Tornado application!"}, - access_token=self.current_user["access_token"]) - - if not new_entry: - # Call failed; perhaps missing permission? - yield self.authorize_redirect() - return - self.finish("Posted a message!") - - .. testoutput:: - :hide: - - The given path is relative to ``self._FACEBOOK_BASE_URL``, - by default "https://graph.facebook.com". - - This method is a wrapper around `OAuth2Mixin.oauth2_request`; - the only difference is that this method takes a relative path, - while ``oauth2_request`` takes a complete url. - - .. versionchanged:: 3.1 - Added the ability to override ``self._FACEBOOK_BASE_URL``. - """ - url = self._FACEBOOK_BASE_URL + path - # Thanks to the _auth_return_future decorator, our "callback" - # argument is a Future, which we cannot pass as a callback to - # oauth2_request. Instead, have oauth2_request return a - # future and chain them together. - oauth_future = self.oauth2_request(url, access_token=access_token, - post_args=post_args, **args) - chain_future(oauth_future, callback) - - -def _oauth_signature(consumer_token, method, url, parameters={}, token=None): - """Calculates the HMAC-SHA1 OAuth signature for the given request. - - See http://oauth.net/core/1.0/#signing_process - """ - parts = urlparse.urlparse(url) - scheme, netloc, path = parts[:3] - normalized_url = scheme.lower() + "://" + netloc.lower() + path - - base_elems = [] - base_elems.append(method.upper()) - base_elems.append(normalized_url) - base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v))) - for k, v in sorted(parameters.items()))) - base_string = "&".join(_oauth_escape(e) for e in base_elems) - - key_elems = [escape.utf8(consumer_token["secret"])] - key_elems.append(escape.utf8(token["secret"] if token else "")) - key = b"&".join(key_elems) - - hash = hmac.new(key, escape.utf8(base_string), hashlib.sha1) - return binascii.b2a_base64(hash.digest())[:-1] - - -def _oauth10a_signature(consumer_token, method, url, parameters={}, token=None): - """Calculates the HMAC-SHA1 OAuth 1.0a signature for the given request. - - See http://oauth.net/core/1.0a/#signing_process - """ - parts = urlparse.urlparse(url) - scheme, netloc, path = parts[:3] - normalized_url = scheme.lower() + "://" + netloc.lower() + path - - base_elems = [] - base_elems.append(method.upper()) - base_elems.append(normalized_url) - base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v))) - for k, v in sorted(parameters.items()))) - - base_string = "&".join(_oauth_escape(e) for e in base_elems) - key_elems = [escape.utf8(urllib_parse.quote(consumer_token["secret"], safe='~'))] - key_elems.append(escape.utf8(urllib_parse.quote(token["secret"], safe='~') if token else "")) - key = b"&".join(key_elems) - - hash = hmac.new(key, escape.utf8(base_string), hashlib.sha1) - return binascii.b2a_base64(hash.digest())[:-1] - - -def _oauth_escape(val): - if isinstance(val, unicode_type): - val = val.encode("utf-8") - return urllib_parse.quote(val, safe="~") - - -def _oauth_parse_response(body): - # I can't find an officially-defined encoding for oauth responses and - # have never seen anyone use non-ascii. Leave the response in a byte - # string for python 2, and use utf8 on python 3. - body = escape.native_str(body) - p = urlparse.parse_qs(body, keep_blank_values=False) - token = dict(key=p["oauth_token"][0], secret=p["oauth_token_secret"][0]) - - # Add the extra parameters the Provider included to the token - special = ("oauth_token", "oauth_token_secret") - token.update((k, p[k][0]) for k in p if k not in special) - return token diff --git a/salt/ext/tornado/autoreload.py b/salt/ext/tornado/autoreload.py deleted file mode 100644 index 4e49a50c17c..00000000000 --- a/salt/ext/tornado/autoreload.py +++ /dev/null @@ -1,335 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Automatically restart the server when a source file is modified. - -Most applications should not access this module directly. Instead, -pass the keyword argument ``autoreload=True`` to the -`tornado.web.Application` constructor (or ``debug=True``, which -enables this setting and several others). This will enable autoreload -mode as well as checking for changes to templates and static -resources. Note that restarting is a destructive operation and any -requests in progress will be aborted when the process restarts. (If -you want to disable autoreload while using other debug-mode features, -pass both ``debug=True`` and ``autoreload=False``). - -This module can also be used as a command-line wrapper around scripts -such as unit test runners. See the `main` method for details. - -The command-line wrapper and Application debug modes can be used together. -This combination is encouraged as the wrapper catches syntax errors and -other import-time failures, while debug mode catches changes once -the server has started. - -This module depends on `.IOLoop`, so it will not work in WSGI applications -and Google App Engine. It also will not work correctly when `.HTTPServer`'s -multi-process mode is used. - -Reloading loses any Python interpreter command-line arguments (e.g. ``-u``) -because it re-executes Python using ``sys.executable`` and ``sys.argv``. -Additionally, modifying these variables will cause reloading to behave -incorrectly. - -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import os -import sys - -# sys.path handling -# ----------------- -# -# If a module is run with "python -m", the current directory (i.e. "") -# is automatically prepended to sys.path, but not if it is run as -# "path/to/file.py". The processing for "-m" rewrites the former to -# the latter, so subsequent executions won't have the same path as the -# original. -# -# Conversely, when run as path/to/file.py, the directory containing -# file.py gets added to the path, which can cause confusion as imports -# may become relative in spite of the future import. -# -# We address the former problem by setting the $PYTHONPATH environment -# variable before re-execution so the new process will see the correct -# path. We attempt to address the latter problem when tornado.autoreload -# is run as __main__, although we can't fix the general case because -# we cannot reliably reconstruct the original command line -# (http://bugs.python.org/issue14208). - -if __name__ == "__main__": - # This sys.path manipulation must come before our imports (as much - # as possible - if we introduced a tornado.sys or tornado.os - # module we'd be in trouble), or else our imports would become - # relative again despite the future import. - # - # There is a separate __main__ block at the end of the file to call main(). - if sys.path[0] == os.path.dirname(__file__): - del sys.path[0] - -import functools -import logging -import os -import pkgutil # type: ignore -import sys -import traceback -import types -import subprocess -import weakref - -from salt.ext.tornado import ioloop -from salt.ext.tornado.log import gen_log -from salt.ext.tornado import process -from salt.ext.tornado.util import exec_in - -try: - import signal -except ImportError: - signal = None - -# os.execv is broken on Windows and can't properly parse command line -# arguments and executable name if they contain whitespaces. subprocess -# fixes that behavior. -_has_execv = sys.platform != 'win32' - -_watched_files = set() -_reload_hooks = [] -_reload_attempted = False -_io_loops = weakref.WeakKeyDictionary() # type: ignore - - -def start(io_loop=None, check_time=500): - """Begins watching source files for changes. - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - """ - io_loop = io_loop or ioloop.IOLoop.current() - if io_loop in _io_loops: - return - _io_loops[io_loop] = True - if len(_io_loops) > 1: - gen_log.warning("tornado.autoreload started more than once in the same process") - modify_times = {} - callback = functools.partial(_reload_on_update, modify_times) - scheduler = ioloop.PeriodicCallback(callback, check_time, io_loop=io_loop) - scheduler.start() - - -def wait(): - """Wait for a watched file to change, then restart the process. - - Intended to be used at the end of scripts like unit test runners, - to run the tests again after any source file changes (but see also - the command-line interface in `main`) - """ - io_loop = ioloop.IOLoop() - start(io_loop) - io_loop.start() - - -def watch(filename): - """Add a file to the watch list. - - All imported modules are watched by default. - """ - _watched_files.add(filename) - - -def add_reload_hook(fn): - """Add a function to be called before reloading the process. - - Note that for open file and socket handles it is generally - preferable to set the ``FD_CLOEXEC`` flag (using `fcntl` or - ``tornado.platform.auto.set_close_exec``) instead - of using a reload hook to close them. - """ - _reload_hooks.append(fn) - - -def _reload_on_update(modify_times): - if _reload_attempted: - # We already tried to reload and it didn't work, so don't try again. - return - if process.task_id() is not None: - # We're in a child process created by fork_processes. If child - # processes restarted themselves, they'd all restart and then - # all call fork_processes again. - return - for module in list(sys.modules.values()): - # Some modules play games with sys.modules (e.g. email/__init__.py - # in the standard library), and occasionally this can cause strange - # failures in getattr. Just ignore anything that's not an ordinary - # module. - if not isinstance(module, types.ModuleType): - continue - path = getattr(module, "__file__", None) - if not path: - continue - if path.endswith(".pyc") or path.endswith(".pyo"): - path = path[:-1] - _check_file(modify_times, path) - for path in _watched_files: - _check_file(modify_times, path) - - -def _check_file(modify_times, path): - try: - modified = os.stat(path).st_mtime - except Exception: - return - if path not in modify_times: - modify_times[path] = modified - return - if modify_times[path] != modified: - gen_log.info("%s modified; restarting server", path) - _reload() - - -def _reload(): - global _reload_attempted - _reload_attempted = True - for fn in _reload_hooks: - fn() - if hasattr(signal, "setitimer"): - # Clear the alarm signal set by - # ioloop.set_blocking_log_threshold so it doesn't fire - # after the exec. - signal.setitimer(signal.ITIMER_REAL, 0, 0) - # sys.path fixes: see comments at top of file. If sys.path[0] is an empty - # string, we were (probably) invoked with -m and the effective path - # is about to change on re-exec. Add the current directory to $PYTHONPATH - # to ensure that the new process sees the same path we did. - path_prefix = '.' + os.pathsep - if (sys.path[0] == '' and - not os.environ.get("PYTHONPATH", "").startswith(path_prefix)): - os.environ["PYTHONPATH"] = (path_prefix + - os.environ.get("PYTHONPATH", "")) - if not _has_execv: - subprocess.Popen([sys.executable] + sys.argv) - sys.exit(0) - else: - try: - os.execv(sys.executable, [sys.executable] + sys.argv) - except OSError: - # Mac OS X versions prior to 10.6 do not support execv in - # a process that contains multiple threads. Instead of - # re-executing in the current process, start a new one - # and cause the current process to exit. This isn't - # ideal since the new process is detached from the parent - # terminal and thus cannot easily be killed with ctrl-C, - # but it's better than not being able to autoreload at - # all. - # Unfortunately the errno returned in this case does not - # appear to be consistent, so we can't easily check for - # this error specifically. - os.spawnv(os.P_NOWAIT, sys.executable, - [sys.executable] + sys.argv) - # At this point the IOLoop has been closed and finally - # blocks will experience errors if we allow the stack to - # unwind, so just exit uncleanly. - os._exit(0) - - -_USAGE = """\ -Usage: - python -m tornado.autoreload -m module.to.run [args...] - python -m tornado.autoreload path/to/script.py [args...] -""" - - -def main(): - """Command-line wrapper to re-run a script whenever its source changes. - - Scripts may be specified by filename or module name:: - - python -m tornado.autoreload -m tornado.test.runtests - python -m tornado.autoreload tornado/test/runtests.py - - Running a script with this wrapper is similar to calling - `tornado.autoreload.wait` at the end of the script, but this wrapper - can catch import-time problems like syntax errors that would otherwise - prevent the script from reaching its call to `wait`. - """ - original_argv = sys.argv - sys.argv = sys.argv[:] - if len(sys.argv) >= 3 and sys.argv[1] == "-m": - mode = "module" - module = sys.argv[2] - del sys.argv[1:3] - elif len(sys.argv) >= 2: - mode = "script" - script = sys.argv[1] - sys.argv = sys.argv[1:] - else: - print(_USAGE, file=sys.stderr) - sys.exit(1) - - try: - if mode == "module": - import runpy - runpy.run_module(module, run_name="__main__", alter_sys=True) - elif mode == "script": - with open(script) as f: - # Execute the script in our namespace instead of creating - # a new one so that something that tries to import __main__ - # (e.g. the unittest module) will see names defined in the - # script instead of just those defined in this module. - global __file__ - __file__ = script - # If __package__ is defined, imports may be incorrectly - # interpreted as relative to this module. - global __package__ - del __package__ - exec_in(f.read(), globals(), globals()) - except SystemExit as e: - logging.basicConfig() - gen_log.info("Script exited with status %s", e.code) - except Exception as e: - logging.basicConfig() - gen_log.warning("Script exited with uncaught exception", exc_info=True) - # If an exception occurred at import time, the file with the error - # never made it into sys.modules and so we won't know to watch it. - # Just to make sure we've covered everything, walk the stack trace - # from the exception and watch every file. - for (filename, lineno, name, line) in traceback.extract_tb(sys.exc_info()[2]): - watch(filename) - if isinstance(e, SyntaxError): - # SyntaxErrors are special: their innermost stack frame is fake - # so extract_tb won't see it and we have to get the filename - # from the exception object. - watch(e.filename) - else: - logging.basicConfig() - gen_log.info("Script exited normally") - # restore sys.argv so subsequent executions will include autoreload - sys.argv = original_argv - - if mode == 'module': - # runpy did a fake import of the module as __main__, but now it's - # no longer in sys.modules. Figure out where it is and watch it. - loader = pkgutil.get_loader(module) - if loader is not None: - watch(loader.get_filename()) - - wait() - - -if __name__ == "__main__": - # See also the other __main__ block at the top of the file, which modifies - # sys.path before our imports - main() diff --git a/salt/ext/tornado/concurrent.py b/salt/ext/tornado/concurrent.py deleted file mode 100644 index bea09ba125e..00000000000 --- a/salt/ext/tornado/concurrent.py +++ /dev/null @@ -1,547 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Utilities for working with threads and ``Futures``. - -``Futures`` are a pattern for concurrent programming introduced in -Python 3.2 in the `concurrent.futures` package. This package defines -a mostly-compatible `Future` class designed for use from coroutines, -as well as some utility functions for interacting with the -`concurrent.futures` package. -""" -# pylint: skip-file -from __future__ import absolute_import, division, print_function - -import functools -import platform -import sys -import textwrap -import traceback - -from salt.ext.tornado.log import app_log -from salt.ext.tornado.stack_context import ExceptionStackContext, wrap -from salt.ext.tornado.util import ArgReplacer, is_finalizing, raise_exc_info - -try: - from concurrent import futures -except ImportError: - futures = None - -try: - import typing -except ImportError: - typing = None - - -# Can the garbage collector handle cycles that include __del__ methods? -# This is true in cpython beginning with version 3.4 (PEP 442). -_GC_CYCLE_FINALIZERS = platform.python_implementation() == "CPython" and sys.version_info >= ( - 3, - 4, -) - - -class ReturnValueIgnoredError(Exception): - pass - - -# This class and associated code in the future object is derived -# from the Trollius project, a backport of asyncio to Python 2.x - 3.x - - -class _TracebackLogger(object): - """Helper to log a traceback upon destruction if not cleared. - - This solves a nasty problem with Futures and Tasks that have an - exception set: if nobody asks for the exception, the exception is - never logged. This violates the Zen of Python: 'Errors should - never pass silently. Unless explicitly silenced.' - - However, we don't want to log the exception as soon as - set_exception() is called: if the calling code is written - properly, it will get the exception and handle it properly. But - we *do* want to log it if result() or exception() was never called - -- otherwise developers waste a lot of time wondering why their - buggy code fails silently. - - An earlier attempt added a __del__() method to the Future class - itself, but this backfired because the presence of __del__() - prevents garbage collection from breaking cycles. A way out of - this catch-22 is to avoid having a __del__() method on the Future - class itself, but instead to have a reference to a helper object - with a __del__() method that logs the traceback, where we ensure - that the helper object doesn't participate in cycles, and only the - Future has a reference to it. - - The helper object is added when set_exception() is called. When - the Future is collected, and the helper is present, the helper - object is also collected, and its __del__() method will log the - traceback. When the Future's result() or exception() method is - called (and a helper object is present), it removes the helper - object, after calling its clear() method to prevent it from - logging. - - One downside is that we do a fair amount of work to extract the - traceback from the exception, even when it is never logged. It - would seem cheaper to just store the exception object, but that - references the traceback, which references stack frames, which may - reference the Future, which references the _TracebackLogger, and - then the _TracebackLogger would be included in a cycle, which is - what we're trying to avoid! As an optimization, we don't - immediately format the exception; we only do the work when - activate() is called, which call is delayed until after all the - Future's callbacks have run. Since usually a Future has at least - one callback (typically set by 'yield From') and usually that - callback extracts the callback, thereby removing the need to - format the exception. - - PS. I don't claim credit for this solution. I first heard of it - in a discussion about closing files when they are collected. - """ - - __slots__ = ("exc_info", "formatted_tb") - - def __init__(self, exc_info): - self.exc_info = exc_info - self.formatted_tb = None - - def activate(self): - exc_info = self.exc_info - if exc_info is not None: - self.exc_info = None - self.formatted_tb = traceback.format_exception(*exc_info) - - def clear(self): - self.exc_info = None - self.formatted_tb = None - - def __del__(self, is_finalizing=is_finalizing): - if not is_finalizing() and self.formatted_tb: - app_log.error( - "Future exception was never retrieved: %s", - "".join(self.formatted_tb).rstrip(), - ) - - -class Future(object): - """Placeholder for an asynchronous result. - - A ``Future`` encapsulates the result of an asynchronous - operation. In synchronous applications ``Futures`` are used - to wait for the result from a thread or process pool; in - Tornado they are normally used with `.IOLoop.add_future` or by - yielding them in a `.gen.coroutine`. - - `tornado.concurrent.Future` is similar to - `concurrent.futures.Future`, but not thread-safe (and therefore - faster for use with single-threaded event loops). - - In addition to ``exception`` and ``set_exception``, methods ``exc_info`` - and ``set_exc_info`` are supported to capture tracebacks in Python 2. - The traceback is automatically available in Python 3, but in the - Python 2 futures backport this information is discarded. - This functionality was previously available in a separate class - ``TracebackFuture``, which is now a deprecated alias for this class. - - .. versionchanged:: 4.0 - `tornado.concurrent.Future` is always a thread-unsafe ``Future`` - with support for the ``exc_info`` methods. Previously it would - be an alias for the thread-safe `concurrent.futures.Future` - if that package was available and fall back to the thread-unsafe - implementation if it was not. - - .. versionchanged:: 4.1 - If a `.Future` contains an error but that error is never observed - (by calling ``result()``, ``exception()``, or ``exc_info()``), - a stack trace will be logged when the `.Future` is garbage collected. - This normally indicates an error in the application, but in cases - where it results in undesired logging it may be necessary to - suppress the logging by ensuring that the exception is observed: - ``f.add_done_callback(lambda f: f.exception())``. - """ - - def __init__(self): - self._done = False - self._result = None - self._exc_info = None - - self._log_traceback = False # Used for Python >= 3.4 - self._tb_logger = None # Used for Python <= 3.3 - - self._callbacks = [] - - # Implement the Python 3.5 Awaitable protocol if possible - # (we can't use return and yield together until py33). - if sys.version_info >= (3, 3): - exec( - textwrap.dedent( - """ - def __await__(self): - return (yield self) - """ - ) - ) - else: - # Py2-compatible version for use with cython. - def __await__(self): - result = yield self - # StopIteration doesn't take args before py33, - # but Cython recognizes the args tuple. - e = StopIteration() - e.args = (result,) - raise e - - def cancel(self): - """Cancel the operation, if possible. - - Tornado ``Futures`` do not support cancellation, so this method always - returns False. - """ - return False - - def cancelled(self): - """Returns True if the operation has been cancelled. - - Tornado ``Futures`` do not support cancellation, so this method - always returns False. - """ - return False - - def running(self): - """Returns True if this operation is currently running.""" - return not self._done - - def done(self): - """Returns True if the future has finished running.""" - return self._done - - def _clear_tb_log(self): - self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None - - def result(self, timeout=None): - """If the operation succeeded, return its result. If it failed, - re-raise its exception. - - This method takes a ``timeout`` argument for compatibility with - `concurrent.futures.Future` but it is an error to call it - before the `Future` is done, so the ``timeout`` is never used. - """ - self._clear_tb_log() - if self._result is not None: - return self._result - if self._exc_info is not None: - try: - raise_exc_info(self._exc_info) - finally: - self = None - self._check_done() - return self._result - - def exception(self, timeout=None): - """If the operation raised an exception, return the `Exception` - object. Otherwise returns None. - - This method takes a ``timeout`` argument for compatibility with - `concurrent.futures.Future` but it is an error to call it - before the `Future` is done, so the ``timeout`` is never used. - """ - self._clear_tb_log() - if self._exc_info is not None: - return self._exc_info[1] - else: - self._check_done() - return None - - def add_done_callback(self, fn): - """Attaches the given callback to the `Future`. - - It will be invoked with the `Future` as its argument when the Future - has finished running and its result is available. In Tornado - consider using `.IOLoop.add_future` instead of calling - `add_done_callback` directly. - """ - if self._done: - fn(self) - else: - self._callbacks.append(fn) - - def set_result(self, result): - """Sets the result of a ``Future``. - - It is undefined to call any of the ``set`` methods more than once - on the same object. - """ - self._result = result - self._set_done() - - def set_exception(self, exception): - """Sets the exception of a ``Future.``""" - self.set_exc_info( - (exception.__class__, exception, getattr(exception, "__traceback__", None)) - ) - - def exc_info(self): - """Returns a tuple in the same format as `sys.exc_info` or None. - - .. versionadded:: 4.0 - """ - self._clear_tb_log() - return self._exc_info - - def set_exc_info(self, exc_info): - """Sets the exception information of a ``Future.`` - - Preserves tracebacks on Python 2. - - .. versionadded:: 4.0 - """ - self._exc_info = exc_info - self._log_traceback = True - if not _GC_CYCLE_FINALIZERS: - self._tb_logger = _TracebackLogger(exc_info) - - try: - self._set_done() - finally: - # Activate the logger after all callbacks have had a - # chance to call result() or exception(). - if self._log_traceback and self._tb_logger is not None: - self._tb_logger.activate() - self._exc_info = exc_info - - def _check_done(self): - if not self._done: - raise Exception("DummyFuture does not support blocking for results") - - def _set_done(self): - self._done = True - for cb in self._callbacks: - try: - cb(self) - except Exception: - app_log.exception("Exception in callback %r for %r", cb, self) - self._callbacks = None - - # On Python 3.3 or older, objects with a destructor part of a reference - # cycle are never destroyed. It's no longer the case on Python 3.4 thanks to - # the PEP 442. - if _GC_CYCLE_FINALIZERS: - - def __del__(self, is_finalizing=is_finalizing): - if is_finalizing() or not self._log_traceback: - # set_exception() was not called, or result() or exception() - # has consumed the exception - return - - tb = traceback.format_exception(*self._exc_info) - - app_log.error( - "Future %r exception was never retrieved: %s", - self, - "".join(tb).rstrip(), - ) - - -TracebackFuture = Future - -if futures is None: - FUTURES = Future # type: typing.Union[type, typing.Tuple[type, ...]] -else: - FUTURES = (futures.Future, Future) - - -def is_future(x): - return isinstance(x, FUTURES) - - -class DummyExecutor(object): - def submit(self, fn, *args, **kwargs): - future = TracebackFuture() - try: - future.set_result(fn(*args, **kwargs)) - except Exception: - future.set_exc_info(sys.exc_info()) - return future - - def shutdown(self, wait=True): - pass - - -dummy_executor = DummyExecutor() - - -def run_on_executor(*args, **kwargs): - """Decorator to run a synchronous method asynchronously on an executor. - - The decorated method may be called with a ``callback`` keyword - argument and returns a future. - - The `.IOLoop` and executor to be used are determined by the ``io_loop`` - and ``executor`` attributes of ``self``. To use different attributes, - pass keyword arguments to the decorator:: - - @run_on_executor(executor='_thread_pool') - def foo(self): - pass - - .. versionchanged:: 4.2 - Added keyword arguments to use alternative attributes. - """ - - def run_on_executor_decorator(fn): - executor = kwargs.get("executor", "executor") - io_loop = kwargs.get("io_loop", "io_loop") - - @functools.wraps(fn) - def wrapper(self, *args, **kwargs): - callback = kwargs.pop("callback", None) - future = getattr(self, executor).submit(fn, self, *args, **kwargs) - if callback: - getattr(self, io_loop).add_future( - future, lambda future: callback(future.result()) - ) - return future - - return wrapper - - if args and kwargs: - raise ValueError("cannot combine positional and keyword args") - if len(args) == 1: - return run_on_executor_decorator(args[0]) - elif len(args) != 0: - raise ValueError("expected 1 argument, got %d", len(args)) - return run_on_executor_decorator - - -_NO_RESULT = object() - - -def return_future(f): - """Decorator to make a function that returns via callback return a - `Future`. - - The wrapped function should take a ``callback`` keyword argument - and invoke it with one argument when it has finished. To signal failure, - the function can simply raise an exception (which will be - captured by the `.StackContext` and passed along to the ``Future``). - - From the caller's perspective, the callback argument is optional. - If one is given, it will be invoked when the function is complete - with `Future.result()` as an argument. If the function fails, the - callback will not be run and an exception will be raised into the - surrounding `.StackContext`. - - If no callback is given, the caller should use the ``Future`` to - wait for the function to complete (perhaps by yielding it in a - `.gen.engine` function, or passing it to `.IOLoop.add_future`). - - Usage: - - .. testcode:: - - @return_future - def future_func(arg1, arg2, callback): - # Do stuff (possibly asynchronous) - callback(result) - - @gen.engine - def caller(callback): - yield future_func(arg1, arg2) - callback() - - .. - - Note that ``@return_future`` and ``@gen.engine`` can be applied to the - same function, provided ``@return_future`` appears first. However, - consider using ``@gen.coroutine`` instead of this combination. - """ - replacer = ArgReplacer(f, "callback") - - @functools.wraps(f) - def wrapper(*args, **kwargs): - future = TracebackFuture() - callback, args, kwargs = replacer.replace( - lambda value=_NO_RESULT: future.set_result(value), args, kwargs - ) - - def handle_error(typ, value, tb): - future.set_exc_info((typ, value, tb)) - return True - - exc_info = None - with ExceptionStackContext(handle_error): - try: - result = f(*args, **kwargs) - if result is not None: - raise ReturnValueIgnoredError( - "@return_future should not be used with functions " - "that return values" - ) - except: - exc_info = sys.exc_info() - raise - if exc_info is not None: - # If the initial synchronous part of f() raised an exception, - # go ahead and raise it to the caller directly without waiting - # for them to inspect the Future. - future.result() - - # If the caller passed in a callback, schedule it to be called - # when the future resolves. It is important that this happens - # just before we return the future, or else we risk confusing - # stack contexts with multiple exceptions (one here with the - # immediate exception, and again when the future resolves and - # the callback triggers its exception by calling future.result()). - if callback is not None: - - def run_callback(future): - result = future.result() - if result is _NO_RESULT: - callback() - else: - callback(future.result()) - - future.add_done_callback(wrap(run_callback)) - return future - - return wrapper - - -def chain_future(a, b): - """Chain two futures together so that when one completes, so does the other. - - The result (success or failure) of ``a`` will be copied to ``b``, unless - ``b`` has already been completed or cancelled by the time ``a`` finishes. - """ - - def copy(future): - assert future is a - if b.done(): - return - if ( - isinstance(a, TracebackFuture) - and isinstance(b, TracebackFuture) - and a.exc_info() is not None - ): - b.set_exc_info(a.exc_info()) - elif a.exception() is not None: - b.set_exception(a.exception()) - else: - b.set_result(a.result()) - - a.add_done_callback(copy) diff --git a/salt/ext/tornado/curl_httpclient.py b/salt/ext/tornado/curl_httpclient.py deleted file mode 100644 index 8652343cf75..00000000000 --- a/salt/ext/tornado/curl_httpclient.py +++ /dev/null @@ -1,515 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Non-blocking HTTP client implementation using pycurl.""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import collections -import functools -import logging -import pycurl # type: ignore -import threading -import time -from io import BytesIO - -from salt.ext.tornado import httputil -from salt.ext.tornado import ioloop -from salt.ext.tornado import stack_context - -from salt.ext.tornado.escape import utf8, native_str -from salt.ext.tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main - -curl_log = logging.getLogger('tornado.curl_httpclient') - - -class CurlAsyncHTTPClient(AsyncHTTPClient): - def initialize(self, io_loop, max_clients=10, defaults=None): - super(CurlAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) - self._multi = pycurl.CurlMulti() - self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) - self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) - self._curls = [self._curl_create() for i in range(max_clients)] - self._free_list = self._curls[:] - self._requests = collections.deque() - self._fds = {} - self._timeout = None - - # libcurl has bugs that sometimes cause it to not report all - # relevant file descriptors and timeouts to TIMERFUNCTION/ - # SOCKETFUNCTION. Mitigate the effects of such bugs by - # forcing a periodic scan of all active requests. - self._force_timeout_callback = ioloop.PeriodicCallback( - self._handle_force_timeout, 1000, io_loop=io_loop) - self._force_timeout_callback.start() - - # Work around a bug in libcurl 7.29.0: Some fields in the curl - # multi object are initialized lazily, and its destructor will - # segfault if it is destroyed without having been used. Add - # and remove a dummy handle to make sure everything is - # initialized. - dummy_curl_handle = pycurl.Curl() - self._multi.add_handle(dummy_curl_handle) - self._multi.remove_handle(dummy_curl_handle) - - def close(self): - self._force_timeout_callback.stop() - if self._timeout is not None: - self.io_loop.remove_timeout(self._timeout) - for curl in self._curls: - curl.close() - self._multi.close() - super(CurlAsyncHTTPClient, self).close() - - def fetch_impl(self, request, callback): - self._requests.append((request, callback)) - self._process_queue() - self._set_timeout(0) - - def _handle_socket(self, event, fd, multi, data): - """Called by libcurl when it wants to change the file descriptors - it cares about. - """ - event_map = { - pycurl.POLL_NONE: ioloop.IOLoop.NONE, - pycurl.POLL_IN: ioloop.IOLoop.READ, - pycurl.POLL_OUT: ioloop.IOLoop.WRITE, - pycurl.POLL_INOUT: ioloop.IOLoop.READ | ioloop.IOLoop.WRITE - } - if event == pycurl.POLL_REMOVE: - if fd in self._fds: - self.io_loop.remove_handler(fd) - del self._fds[fd] - else: - ioloop_event = event_map[event] - # libcurl sometimes closes a socket and then opens a new - # one using the same FD without giving us a POLL_NONE in - # between. This is a problem with the epoll IOLoop, - # because the kernel can tell when a socket is closed and - # removes it from the epoll automatically, causing future - # update_handler calls to fail. Since we can't tell when - # this has happened, always use remove and re-add - # instead of update. - if fd in self._fds: - self.io_loop.remove_handler(fd) - self.io_loop.add_handler(fd, self._handle_events, - ioloop_event) - self._fds[fd] = ioloop_event - - def _set_timeout(self, msecs): - """Called by libcurl to schedule a timeout.""" - if self._timeout is not None: - self.io_loop.remove_timeout(self._timeout) - self._timeout = self.io_loop.add_timeout( - self.io_loop.time() + msecs / 1000.0, self._handle_timeout) - - def _handle_events(self, fd, events): - """Called by IOLoop when there is activity on one of our - file descriptors. - """ - action = 0 - if events & ioloop.IOLoop.READ: - action |= pycurl.CSELECT_IN - if events & ioloop.IOLoop.WRITE: - action |= pycurl.CSELECT_OUT - while True: - try: - ret, num_handles = self._multi.socket_action(fd, action) - except pycurl.error as e: - ret = e.args[0] - if ret != pycurl.E_CALL_MULTI_PERFORM: - break - self._finish_pending_requests() - - def _handle_timeout(self): - """Called by IOLoop when the requested timeout has passed.""" - with stack_context.NullContext(): - self._timeout = None - while True: - try: - ret, num_handles = self._multi.socket_action( - pycurl.SOCKET_TIMEOUT, 0) - except pycurl.error as e: - ret = e.args[0] - if ret != pycurl.E_CALL_MULTI_PERFORM: - break - self._finish_pending_requests() - - # In theory, we shouldn't have to do this because curl will - # call _set_timeout whenever the timeout changes. However, - # sometimes after _handle_timeout we will need to reschedule - # immediately even though nothing has changed from curl's - # perspective. This is because when socket_action is - # called with SOCKET_TIMEOUT, libcurl decides internally which - # timeouts need to be processed by using a monotonic clock - # (where available) while tornado uses python's time.time() - # to decide when timeouts have occurred. When those clocks - # disagree on elapsed time (as they will whenever there is an - # NTP adjustment), tornado might call _handle_timeout before - # libcurl is ready. After each timeout, resync the scheduled - # timeout with libcurl's current state. - new_timeout = self._multi.timeout() - if new_timeout >= 0: - self._set_timeout(new_timeout) - - def _handle_force_timeout(self): - """Called by IOLoop periodically to ask libcurl to process any - events it may have forgotten about. - """ - with stack_context.NullContext(): - while True: - try: - ret, num_handles = self._multi.socket_all() - except pycurl.error as e: - ret = e.args[0] - if ret != pycurl.E_CALL_MULTI_PERFORM: - break - self._finish_pending_requests() - - def _finish_pending_requests(self): - """Process any requests that were completed by the last - call to multi.socket_action. - """ - while True: - num_q, ok_list, err_list = self._multi.info_read() - for curl in ok_list: - self._finish(curl) - for curl, errnum, errmsg in err_list: - self._finish(curl, errnum, errmsg) - if num_q == 0: - break - self._process_queue() - - def _process_queue(self): - with stack_context.NullContext(): - while True: - started = 0 - while self._free_list and self._requests: - started += 1 - curl = self._free_list.pop() - (request, callback) = self._requests.popleft() - curl.info = { - "headers": httputil.HTTPHeaders(), - "buffer": BytesIO(), - "request": request, - "callback": callback, - "curl_start_time": time.time(), - } - try: - self._curl_setup_request( - curl, request, curl.info["buffer"], - curl.info["headers"]) - except Exception as e: - # If there was an error in setup, pass it on - # to the callback. Note that allowing the - # error to escape here will appear to work - # most of the time since we are still in the - # caller's original stack frame, but when - # _process_queue() is called from - # _finish_pending_requests the exceptions have - # nowhere to go. - self._free_list.append(curl) - callback(HTTPResponse( - request=request, - code=599, - error=e)) - else: - self._multi.add_handle(curl) - - if not started: - break - - def _finish(self, curl, curl_error=None, curl_message=None): - info = curl.info - curl.info = None - self._multi.remove_handle(curl) - self._free_list.append(curl) - buffer = info["buffer"] - if curl_error: - error = CurlError(curl_error, curl_message) - code = error.code - effective_url = None - buffer.close() - buffer = None - else: - error = None - code = curl.getinfo(pycurl.HTTP_CODE) - effective_url = curl.getinfo(pycurl.EFFECTIVE_URL) - buffer.seek(0) - # the various curl timings are documented at - # http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html - time_info = dict( - queue=info["curl_start_time"] - info["request"].start_time, - namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME), - connect=curl.getinfo(pycurl.CONNECT_TIME), - pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME), - starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME), - total=curl.getinfo(pycurl.TOTAL_TIME), - redirect=curl.getinfo(pycurl.REDIRECT_TIME), - ) - try: - info["callback"](HTTPResponse( - request=info["request"], code=code, headers=info["headers"], - buffer=buffer, effective_url=effective_url, error=error, - reason=info['headers'].get("X-Http-Reason", None), - request_time=time.time() - info["curl_start_time"], - time_info=time_info)) - except Exception: - self.handle_callback_exception(info["callback"]) - - def handle_callback_exception(self, callback): - self.io_loop.handle_callback_exception(callback) - - def _curl_create(self): - curl = pycurl.Curl() - if curl_log.isEnabledFor(logging.DEBUG): - curl.setopt(pycurl.VERBOSE, 1) - curl.setopt(pycurl.DEBUGFUNCTION, self._curl_debug) - if hasattr(pycurl, 'PROTOCOLS'): # PROTOCOLS first appeared in pycurl 7.19.5 (2014-07-12) - curl.setopt(pycurl.PROTOCOLS, pycurl.PROTO_HTTP | pycurl.PROTO_HTTPS) - curl.setopt(pycurl.REDIR_PROTOCOLS, pycurl.PROTO_HTTP | pycurl.PROTO_HTTPS) - return curl - - def _curl_setup_request(self, curl, request, buffer, headers): - curl.setopt(pycurl.URL, native_str(request.url)) - - # libcurl's magic "Expect: 100-continue" behavior causes delays - # with servers that don't support it (which include, among others, - # Google's OpenID endpoint). Additionally, this behavior has - # a bug in conjunction with the curl_multi_socket_action API - # (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976), - # which increases the delays. It's more trouble than it's worth, - # so just turn off the feature (yes, setting Expect: to an empty - # value is the official way to disable this) - if "Expect" not in request.headers: - request.headers["Expect"] = "" - - # libcurl adds Pragma: no-cache by default; disable that too - if "Pragma" not in request.headers: - request.headers["Pragma"] = "" - - curl.setopt(pycurl.HTTPHEADER, - ["%s: %s" % (native_str(k), native_str(v)) - for k, v in request.headers.get_all()]) - - curl.setopt(pycurl.HEADERFUNCTION, - functools.partial(self._curl_header_callback, - headers, request.header_callback)) - if request.streaming_callback: - def write_function(chunk): - self.io_loop.add_callback(request.streaming_callback, chunk) - else: - write_function = buffer.write - if bytes is str: # py2 - curl.setopt(pycurl.WRITEFUNCTION, write_function) - else: # py3 - # Upstream pycurl doesn't support py3, but ubuntu 12.10 includes - # a fork/port. That version has a bug in which it passes unicode - # strings instead of bytes to the WRITEFUNCTION. This means that - # if you use a WRITEFUNCTION (which tornado always does), you cannot - # download arbitrary binary data. This needs to be fixed in the - # ported pycurl package, but in the meantime this lambda will - # make it work for downloading (utf8) text. - curl.setopt(pycurl.WRITEFUNCTION, lambda s: write_function(utf8(s))) - curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) - curl.setopt(pycurl.MAXREDIRS, request.max_redirects) - curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout)) - curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout)) - if request.user_agent: - curl.setopt(pycurl.USERAGENT, native_str(request.user_agent)) - else: - curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)") - if request.network_interface: - curl.setopt(pycurl.INTERFACE, request.network_interface) - if request.decompress_response: - curl.setopt(pycurl.ENCODING, "gzip,deflate") - else: - curl.setopt(pycurl.ENCODING, "none") - if request.proxy_host and request.proxy_port: - curl.setopt(pycurl.PROXY, request.proxy_host) - curl.setopt(pycurl.PROXYPORT, request.proxy_port) - if request.proxy_username: - credentials = '%s:%s' % (request.proxy_username, - request.proxy_password) - curl.setopt(pycurl.PROXYUSERPWD, credentials) - - if (request.proxy_auth_mode is None or - request.proxy_auth_mode == "basic"): - curl.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_BASIC) - elif request.proxy_auth_mode == "digest": - curl.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_DIGEST) - else: - raise ValueError( - "Unsupported proxy_auth_mode %s" % request.proxy_auth_mode) - else: - curl.setopt(pycurl.PROXY, '') - curl.unsetopt(pycurl.PROXYUSERPWD) - if request.validate_cert: - curl.setopt(pycurl.SSL_VERIFYPEER, 1) - curl.setopt(pycurl.SSL_VERIFYHOST, 2) - else: - curl.setopt(pycurl.SSL_VERIFYPEER, 0) - curl.setopt(pycurl.SSL_VERIFYHOST, 0) - if request.ca_certs is not None: - curl.setopt(pycurl.CAINFO, request.ca_certs) - else: - # There is no way to restore pycurl.CAINFO to its default value - # (Using unsetopt makes it reject all certificates). - # I don't see any way to read the default value from python so it - # can be restored later. We'll have to just leave CAINFO untouched - # if no ca_certs file was specified, and require that if any - # request uses a custom ca_certs file, they all must. - pass - - if request.allow_ipv6 is False: - # Curl behaves reasonably when DNS resolution gives an ipv6 address - # that we can't reach, so allow ipv6 unless the user asks to disable. - curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) - else: - curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER) - - # Set the request method through curl's irritating interface which makes - # up names for almost every single method - curl_options = { - "GET": pycurl.HTTPGET, - "POST": pycurl.POST, - "PUT": pycurl.UPLOAD, - "HEAD": pycurl.NOBODY, - } - custom_methods = set(["DELETE", "OPTIONS", "PATCH"]) - for o in curl_options.values(): - curl.setopt(o, False) - if request.method in curl_options: - curl.unsetopt(pycurl.CUSTOMREQUEST) - curl.setopt(curl_options[request.method], True) - elif request.allow_nonstandard_methods or request.method in custom_methods: - curl.setopt(pycurl.CUSTOMREQUEST, request.method) - else: - raise KeyError('unknown method ' + request.method) - - body_expected = request.method in ("POST", "PATCH", "PUT") - body_present = request.body is not None - if not request.allow_nonstandard_methods: - # Some HTTP methods nearly always have bodies while others - # almost never do. Fail in this case unless the user has - # opted out of sanity checks with allow_nonstandard_methods. - if ((body_expected and not body_present) or - (body_present and not body_expected)): - raise ValueError( - 'Body must %sbe None for method %s (unless ' - 'allow_nonstandard_methods is true)' % - ('not ' if body_expected else '', request.method)) - - if body_expected or body_present: - if request.method == "GET": - # Even with `allow_nonstandard_methods` we disallow - # GET with a body (because libcurl doesn't allow it - # unless we use CUSTOMREQUEST). While the spec doesn't - # forbid clients from sending a body, it arguably - # disallows the server from doing anything with them. - raise ValueError('Body must be None for GET request') - request_buffer = BytesIO(utf8(request.body or '')) - - def ioctl(cmd): - if cmd == curl.IOCMD_RESTARTREAD: - request_buffer.seek(0) - curl.setopt(pycurl.READFUNCTION, request_buffer.read) - curl.setopt(pycurl.IOCTLFUNCTION, ioctl) - if request.method == "POST": - curl.setopt(pycurl.POSTFIELDSIZE, len(request.body or '')) - else: - curl.setopt(pycurl.UPLOAD, True) - curl.setopt(pycurl.INFILESIZE, len(request.body or '')) - - if request.auth_username is not None: - userpwd = "%s:%s" % (request.auth_username, request.auth_password or '') - - if request.auth_mode is None or request.auth_mode == "basic": - curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) - elif request.auth_mode == "digest": - curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST) - else: - raise ValueError("Unsupported auth_mode %s" % request.auth_mode) - - curl.setopt(pycurl.USERPWD, native_str(userpwd)) - curl_log.debug("%s %s (username: %r)", request.method, request.url, - request.auth_username) - else: - curl.unsetopt(pycurl.USERPWD) - curl_log.debug("%s %s", request.method, request.url) - - if request.client_cert is not None: - curl.setopt(pycurl.SSLCERT, request.client_cert) - - if request.client_key is not None: - curl.setopt(pycurl.SSLKEY, request.client_key) - - if request.ssl_options is not None: - raise ValueError("ssl_options not supported in curl_httpclient") - - if threading.activeCount() > 1: - # libcurl/pycurl is not thread-safe by default. When multiple threads - # are used, signals should be disabled. This has the side effect - # of disabling DNS timeouts in some environments (when libcurl is - # not linked against ares), so we don't do it when there is only one - # thread. Applications that use many short-lived threads may need - # to set NOSIGNAL manually in a prepare_curl_callback since - # there may not be any other threads running at the time we call - # threading.activeCount. - curl.setopt(pycurl.NOSIGNAL, 1) - if request.prepare_curl_callback is not None: - request.prepare_curl_callback(curl) - - def _curl_header_callback(self, headers, header_callback, header_line): - header_line = native_str(header_line.decode('latin1')) - if header_callback is not None: - self.io_loop.add_callback(header_callback, header_line) - # header_line as returned by curl includes the end-of-line characters. - # whitespace at the start should be preserved to allow multi-line headers - header_line = header_line.rstrip() - if header_line.startswith("HTTP/"): - headers.clear() - try: - (__, __, reason) = httputil.parse_response_start_line(header_line) - header_line = "X-Http-Reason: %s" % reason - except httputil.HTTPInputError: - return - if not header_line: - return - headers.parse_line(header_line) - - def _curl_debug(self, debug_type, debug_msg): - debug_types = ('I', '<', '>', '<', '>') - debug_msg = native_str(debug_msg) - if debug_type == 0: - curl_log.debug('%s', debug_msg.strip()) - elif debug_type in (1, 2): - for line in debug_msg.splitlines(): - curl_log.debug('%s %s', debug_types[debug_type], line) - elif debug_type == 4: - curl_log.debug('%s %r', debug_types[debug_type], debug_msg) - - -class CurlError(HTTPError): - def __init__(self, errno, message): - HTTPError.__init__(self, 599, message) - self.errno = errno - - -if __name__ == "__main__": - AsyncHTTPClient.configure(CurlAsyncHTTPClient) - main() diff --git a/salt/ext/tornado/escape.py b/salt/ext/tornado/escape.py deleted file mode 100644 index e609d8d8ded..00000000000 --- a/salt/ext/tornado/escape.py +++ /dev/null @@ -1,399 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Escaping/unescaping methods for HTML, JSON, URLs, and others. - -Also includes a few other miscellaneous string manipulation functions that -have crept in over time. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import json -import re - -from salt.ext.tornado.util import PY3, unicode_type, basestring_type - -if PY3: - from urllib.parse import parse_qs as _parse_qs - import html.entities as htmlentitydefs - import urllib.parse as urllib_parse - unichr = chr -else: - from urlparse import parse_qs as _parse_qs - import htmlentitydefs - import urllib as urllib_parse - -try: - import typing # noqa -except ImportError: - pass - - -_XHTML_ESCAPE_RE = re.compile('[&<>"\']') -_XHTML_ESCAPE_DICT = {'&': '&', '<': '<', '>': '>', '"': '"', - '\'': '''} - - -def xhtml_escape(value): - """Escapes a string so it is valid within HTML or XML. - - Escapes the characters ``<``, ``>``, ``"``, ``'``, and ``&``. - When used in attribute values the escaped strings must be enclosed - in quotes. - - .. versionchanged:: 3.2 - - Added the single quote to the list of escaped characters. - """ - return _XHTML_ESCAPE_RE.sub(lambda match: _XHTML_ESCAPE_DICT[match.group(0)], - to_basestring(value)) - - -def xhtml_unescape(value): - """Un-escapes an XML-escaped string.""" - return re.sub(r"&(#?)(\w+?);", _convert_entity, _unicode(value)) - - -# The fact that json_encode wraps json.dumps is an implementation detail. -# Please see https://github.com/tornadoweb/tornado/pull/706 -# before sending a pull request that adds **kwargs to this function. -def json_encode(value): - """JSON-encodes the given Python object.""" - # JSON permits but does not require forward slashes to be escaped. - # This is useful when json data is emitted in a tags from prematurely terminating - # the javascript. Some json libraries do this escaping by default, - # although python's standard library does not, so we do it here. - # http://stackoverflow.com/questions/1580647/json-why-are-forward-slashes-escaped - return json.dumps(value).replace("typing.Union[bytes,None] - """Converts a string argument to a byte string. - - If the argument is already a byte string or None, it is returned unchanged. - Otherwise it must be a unicode string and is encoded as utf8. - """ - if isinstance(value, _UTF8_TYPES): - return value - if not isinstance(value, unicode_type): - raise TypeError( - "Expected bytes, unicode, or None; got %r" % type(value) - ) - return value.encode("utf-8") - - -_TO_UNICODE_TYPES = (unicode_type, type(None)) - - -def to_unicode(value): - """Converts a string argument to a unicode string. - - If the argument is already a unicode string or None, it is returned - unchanged. Otherwise it must be a byte string and is decoded as utf8. - """ - if isinstance(value, _TO_UNICODE_TYPES): - return value - if not isinstance(value, bytes): - raise TypeError( - "Expected bytes, unicode, or None; got %r" % type(value) - ) - return value.decode("utf-8") - - -# to_unicode was previously named _unicode not because it was private, -# but to avoid conflicts with the built-in unicode() function/type -_unicode = to_unicode - -# When dealing with the standard library across python 2 and 3 it is -# sometimes useful to have a direct conversion to the native string type -if str is unicode_type: - native_str = to_unicode -else: - native_str = utf8 - -_BASESTRING_TYPES = (basestring_type, type(None)) - - -def to_basestring(value): - """Converts a string argument to a subclass of basestring. - - In python2, byte and unicode strings are mostly interchangeable, - so functions that deal with a user-supplied argument in combination - with ascii string constants can use either and should return the type - the user supplied. In python3, the two types are not interchangeable, - so this method is needed to convert byte strings to unicode. - """ - if isinstance(value, _BASESTRING_TYPES): - return value - if not isinstance(value, bytes): - raise TypeError( - "Expected bytes, unicode, or None; got %r" % type(value) - ) - return value.decode("utf-8") - - -def recursive_unicode(obj): - """Walks a simple data structure, converting byte strings to unicode. - - Supports lists, tuples, and dictionaries. - """ - if isinstance(obj, dict): - return dict((recursive_unicode(k), recursive_unicode(v)) for (k, v) in obj.items()) - elif isinstance(obj, list): - return list(recursive_unicode(i) for i in obj) - elif isinstance(obj, tuple): - return tuple(recursive_unicode(i) for i in obj) - elif isinstance(obj, bytes): - return to_unicode(obj) - else: - return obj - - -# I originally used the regex from -# http://daringfireball.net/2010/07/improved_regex_for_matching_urls -# but it gets all exponential on certain patterns (such as too many trailing -# dots), causing the regex matcher to never return. -# This regex should avoid those problems. -# Use to_unicode instead of tornado.util.u - we don't want backslashes getting -# processed as escapes. -_URL_RE = re.compile(to_unicode(r"""\b((?:([\w-]+):(/{1,3})|www[.])(?:(?:(?:[^\s&()]|&|")*(?:[^!"#$%&'()*+,.:;<=>?@\[\]^`{|}~\s]))|(?:\((?:[^\s&()]|&|")*\)))+)""")) - - -def linkify(text, shorten=False, extra_params="", - require_protocol=False, permitted_protocols=["http", "https"]): - """Converts plain text into HTML with links. - - For example: ``linkify("Hello http://tornadoweb.org!")`` would return - ``Hello http://tornadoweb.org!`` - - Parameters: - - * ``shorten``: Long urls will be shortened for display. - - * ``extra_params``: Extra text to include in the link tag, or a callable - taking the link as an argument and returning the extra text - e.g. ``linkify(text, extra_params='rel="nofollow" class="external"')``, - or:: - - def extra_params_cb(url): - if url.startswith("http://example.com"): - return 'class="internal"' - else: - return 'class="external" rel="nofollow"' - linkify(text, extra_params=extra_params_cb) - - * ``require_protocol``: Only linkify urls which include a protocol. If - this is False, urls such as www.facebook.com will also be linkified. - - * ``permitted_protocols``: List (or set) of protocols which should be - linkified, e.g. ``linkify(text, permitted_protocols=["http", "ftp", - "mailto"])``. It is very unsafe to include protocols such as - ``javascript``. - """ - if extra_params and not callable(extra_params): - extra_params = " " + extra_params.strip() - - def make_link(m): - url = m.group(1) - proto = m.group(2) - if require_protocol and not proto: - return url # not protocol, no linkify - - if proto and proto not in permitted_protocols: - return url # bad protocol, no linkify - - href = m.group(1) - if not proto: - href = "http://" + href # no proto specified, use http - - if callable(extra_params): - params = " " + extra_params(href).strip() - else: - params = extra_params - - # clip long urls. max_len is just an approximation - max_len = 30 - if shorten and len(url) > max_len: - before_clip = url - if proto: - proto_len = len(proto) + 1 + len(m.group(3) or "") # +1 for : - else: - proto_len = 0 - - parts = url[proto_len:].split("/") - if len(parts) > 1: - # Grab the whole host part plus the first bit of the path - # The path is usually not that interesting once shortened - # (no more slug, etc), so it really just provides a little - # extra indication of shortening. - url = url[:proto_len] + parts[0] + "/" + \ - parts[1][:8].split('?')[0].split('.')[0] - - if len(url) > max_len * 1.5: # still too long - url = url[:max_len] - - if url != before_clip: - amp = url.rfind('&') - # avoid splitting html char entities - if amp > max_len - 5: - url = url[:amp] - url += "..." - - if len(url) >= len(before_clip): - url = before_clip - else: - # full url is visible on mouse-over (for those who don't - # have a status bar, such as Safari by default) - params += ' title="%s"' % href - - return u'%s' % (href, params, url) - - # First HTML-escape so that our strings are all safe. - # The regex is modified to avoid character entites other than & so - # that we won't pick up ", etc. - text = _unicode(xhtml_escape(text)) - return _URL_RE.sub(make_link, text) - - -def _convert_entity(m): - if m.group(1) == "#": - try: - if m.group(2)[:1].lower() == 'x': - return unichr(int(m.group(2)[1:], 16)) - else: - return unichr(int(m.group(2))) - except ValueError: - return "&#%s;" % m.group(2) - try: - return _HTML_UNICODE_MAP[m.group(2)] - except KeyError: - return "&%s;" % m.group(2) - - -def _build_unicode_map(): - unicode_map = {} - for name, value in htmlentitydefs.name2codepoint.items(): - unicode_map[name] = unichr(value) - return unicode_map - - -_HTML_UNICODE_MAP = _build_unicode_map() diff --git a/salt/ext/tornado/gen.py b/salt/ext/tornado/gen.py deleted file mode 100644 index 72f422ce28f..00000000000 --- a/salt/ext/tornado/gen.py +++ /dev/null @@ -1,1304 +0,0 @@ -"""``tornado.gen`` is a generator-based interface to make it easier to -work in an asynchronous environment. Code using the ``gen`` module -is technically asynchronous, but it is written as a single generator -instead of a collection of separate functions. - -For example, the following asynchronous handler: - -.. testcode:: - - class AsyncHandler(RequestHandler): - @asynchronous - def get(self): - http_client = AsyncHTTPClient() - http_client.fetch("http://example.com", - callback=self.on_fetch) - - def on_fetch(self, response): - do_something_with_response(response) - self.render("template.html") - -.. testoutput:: - :hide: - -could be written with ``gen`` as: - -.. testcode:: - - class GenAsyncHandler(RequestHandler): - @gen.coroutine - def get(self): - http_client = AsyncHTTPClient() - response = yield http_client.fetch("http://example.com") - do_something_with_response(response) - self.render("template.html") - -.. testoutput:: - :hide: - -Most asynchronous functions in Tornado return a `.Future`; -yielding this object returns its `~.Future.result`. - -You can also yield a list or dict of ``Futures``, which will be -started at the same time and run in parallel; a list or dict of results will -be returned when they are all finished: - -.. testcode:: - - @gen.coroutine - def get(self): - http_client = AsyncHTTPClient() - response1, response2 = yield [http_client.fetch(url1), - http_client.fetch(url2)] - response_dict = yield dict(response3=http_client.fetch(url3), - response4=http_client.fetch(url4)) - response3 = response_dict['response3'] - response4 = response_dict['response4'] - -.. testoutput:: - :hide: - -If the `~functools.singledispatch` library is available (standard in -Python 3.4, available via the `singledispatch -`_ package on older -versions), additional types of objects may be yielded. Tornado includes -support for ``asyncio.Future`` and Twisted's ``Deferred`` class when -``tornado.platform.asyncio`` and ``tornado.platform.twisted`` are imported. -See the `convert_yielded` function to extend this mechanism. - -.. versionchanged:: 3.2 - Dict support added. - -.. versionchanged:: 4.1 - Support added for yielding ``asyncio`` Futures and Twisted Deferreds - via ``singledispatch``. - -""" -# pylint: skip-file -from __future__ import absolute_import, division, print_function - -import collections -import functools -import itertools -import os -import sys -import textwrap -import types -import weakref - -import salt.ext.tornado as tornado -from salt.ext.tornado.concurrent import Future, TracebackFuture, is_future, chain_future -from salt.ext.tornado.ioloop import IOLoop -from salt.ext.tornado.log import app_log -from salt.ext.tornado import stack_context -from salt.ext.tornado.util import PY3, raise_exc_info - -try: - try: - # py34+ - from functools import singledispatch # type: ignore - except ImportError: - from singledispatch import singledispatch # backport -except ImportError: - # In most cases, singledispatch is required (to avoid - # difficult-to-diagnose problems in which the functionality - # available differs depending on which invisble packages are - # installed). However, in Google App Engine third-party - # dependencies are more trouble so we allow this module to be - # imported without it. - if 'APPENGINE_RUNTIME' not in os.environ: - raise - singledispatch = None - -try: - try: - # py35+ - from collections.abc import Generator as GeneratorType # type: ignore - except ImportError: - from salt.ext.backports_abc import Generator as GeneratorType # type: ignore - - try: - # py35+ - from inspect import isawaitable # type: ignore - except ImportError: - from salt.ext.backports_abc import isawaitable -except ImportError: - if 'APPENGINE_RUNTIME' not in os.environ: - raise - from types import GeneratorType - - def isawaitable(x): # type: ignore - return False - -if PY3: - import builtins -else: - import __builtin__ as builtins - - -class KeyReuseError(Exception): - pass - - -class UnknownKeyError(Exception): - pass - - -class LeakedCallbackError(Exception): - pass - - -class BadYieldError(Exception): - pass - - -class ReturnValueIgnoredError(Exception): - pass - - -class TimeoutError(Exception): - """Exception raised by ``with_timeout``.""" - - -def _value_from_stopiteration(e): - try: - # StopIteration has a value attribute beginning in py33. - # So does our Return class. - return e.value - except AttributeError: - pass - try: - # Cython backports coroutine functionality by putting the value in - # e.args[0]. - return e.args[0] - except (AttributeError, IndexError): - return None - - -def engine(func): - """Callback-oriented decorator for asynchronous generators. - - This is an older interface; for new code that does not need to be - compatible with versions of Tornado older than 3.0 the - `coroutine` decorator is recommended instead. - - This decorator is similar to `coroutine`, except it does not - return a `.Future` and the ``callback`` argument is not treated - specially. - - In most cases, functions decorated with `engine` should take - a ``callback`` argument and invoke it with their result when - they are finished. One notable exception is the - `~tornado.web.RequestHandler` :ref:`HTTP verb methods `, - which use ``self.finish()`` in place of a callback argument. - """ - func = _make_coroutine_wrapper(func, replace_callback=False) - - @functools.wraps(func) - def wrapper(*args, **kwargs): - future = func(*args, **kwargs) - - def final_callback(future): - if future.result() is not None: - raise ReturnValueIgnoredError( - "@gen.engine functions cannot return values: %r" % - (future.result(),)) - # The engine interface doesn't give us any way to return - # errors but to raise them into the stack context. - # Save the stack context here to use when the Future has resolved. - future.add_done_callback(stack_context.wrap(final_callback)) - return wrapper - - -def coroutine(func, replace_callback=True): - """Decorator for asynchronous generators. - - Any generator that yields objects from this module must be wrapped - in either this decorator or `engine`. - - Coroutines may "return" by raising the special exception - `Return(value) `. In Python 3.3+, it is also possible for - the function to simply use the ``return value`` statement (prior to - Python 3.3 generators were not allowed to also return values). - In all versions of Python a coroutine that simply wishes to exit - early may use the ``return`` statement without a value. - - Functions with this decorator return a `.Future`. Additionally, - they may be called with a ``callback`` keyword argument, which - will be invoked with the future's result when it resolves. If the - coroutine fails, the callback will not be run and an exception - will be raised into the surrounding `.StackContext`. The - ``callback`` argument is not visible inside the decorated - function; it is handled by the decorator itself. - - From the caller's perspective, ``@gen.coroutine`` is similar to - the combination of ``@return_future`` and ``@gen.engine``. - - .. warning:: - - When exceptions occur inside a coroutine, the exception - information will be stored in the `.Future` object. You must - examine the result of the `.Future` object, or the exception - may go unnoticed by your code. This means yielding the function - if called from another coroutine, using something like - `.IOLoop.run_sync` for top-level calls, or passing the `.Future` - to `.IOLoop.add_future`. - - """ - return _make_coroutine_wrapper(func, replace_callback=True) - - -# Ties lifetime of runners to their result futures. Github Issue #1769 -# Generators, like any object in Python, must be strong referenced -# in order to not be cleaned up by the garbage collector. When using -# coroutines, the Runner object is what strong-refs the inner -# generator. However, the only item that strong-reffed the Runner -# was the last Future that the inner generator yielded (via the -# Future's internal done_callback list). Usually this is enough, but -# it is also possible for this Future to not have any strong references -# other than other objects referenced by the Runner object (usually -# when using other callback patterns and/or weakrefs). In this -# situation, if a garbage collection ran, a cycle would be detected and -# Runner objects could be destroyed along with their inner generators -# and everything in their local scope. -# This map provides strong references to Runner objects as long as -# their result future objects also have strong references (typically -# from the parent coroutine's Runner). This keeps the coroutine's -# Runner alive. -_futures_to_runners = weakref.WeakKeyDictionary() - - -def _make_coroutine_wrapper(func, replace_callback): - """The inner workings of ``@gen.coroutine`` and ``@gen.engine``. - - The two decorators differ in their treatment of the ``callback`` - argument, so we cannot simply implement ``@engine`` in terms of - ``@coroutine``. - """ - # On Python 3.5, set the coroutine flag on our generator, to allow it - # to be used with 'await'. - wrapped = func - if hasattr(types, 'coroutine'): - func = types.coroutine(func) - - @functools.wraps(wrapped) - def wrapper(*args, **kwargs): - future = TracebackFuture() - - if replace_callback and 'callback' in kwargs: - callback = kwargs.pop('callback') - IOLoop.current().add_future( - future, lambda future: callback(future.result())) - - try: - result = func(*args, **kwargs) - except (Return, StopIteration) as e: - result = _value_from_stopiteration(e) - except Exception: - future.set_exc_info(sys.exc_info()) - return future - else: - if isinstance(result, GeneratorType): - # Inline the first iteration of Runner.run. This lets us - # avoid the cost of creating a Runner when the coroutine - # never actually yields, which in turn allows us to - # use "optional" coroutines in critical path code without - # performance penalty for the synchronous case. - try: - orig_stack_contexts = stack_context._state.contexts - yielded = next(result) - if stack_context._state.contexts is not orig_stack_contexts: - yielded = TracebackFuture() - yielded.set_exception( - stack_context.StackContextInconsistentError( - 'stack_context inconsistency (probably caused ' - 'by yield within a "with StackContext" block)')) - except (StopIteration, Return) as e: - future.set_result(_value_from_stopiteration(e)) - except Exception: - future.set_exc_info(sys.exc_info()) - else: - _futures_to_runners[future] = Runner(result, future, yielded) - yielded = None - try: - return future - finally: - # Subtle memory optimization: if next() raised an exception, - # the future's exc_info contains a traceback which - # includes this stack frame. This creates a cycle, - # which will be collected at the next full GC but has - # been shown to greatly increase memory usage of - # benchmarks (relative to the refcount-based scheme - # used in the absence of cycles). We can avoid the - # cycle by clearing the local variable after we return it. - future = None - future.set_result(result) - return future - - wrapper.__wrapped__ = wrapped - wrapper.__tornado_coroutine__ = True - return wrapper - - -def is_coroutine_function(func): - """Return whether *func* is a coroutine function, i.e. a function - wrapped with `~.gen.coroutine`. - - .. versionadded:: 4.5 - """ - return getattr(func, '__tornado_coroutine__', False) - - -class Return(Exception): - """Special exception to return a value from a `coroutine`. - - If this exception is raised, its value argument is used as the - result of the coroutine:: - - @gen.coroutine - def fetch_json(url): - response = yield AsyncHTTPClient().fetch(url) - raise gen.Return(json_decode(response.body)) - - In Python 3.3, this exception is no longer necessary: the ``return`` - statement can be used directly to return a value (previously - ``yield`` and ``return`` with a value could not be combined in the - same function). - - By analogy with the return statement, the value argument is optional, - but it is never necessary to ``raise gen.Return()``. The ``return`` - statement can be used with no arguments instead. - """ - def __init__(self, value=None): - super(Return, self).__init__() - self.value = value - # Cython recognizes subclasses of StopIteration with a .args tuple. - self.args = (value,) - - -class WaitIterator(object): - """Provides an iterator to yield the results of futures as they finish. - - Yielding a set of futures like this: - - ``results = yield [future1, future2]`` - - pauses the coroutine until both ``future1`` and ``future2`` - return, and then restarts the coroutine with the results of both - futures. If either future is an exception, the expression will - raise that exception and all the results will be lost. - - If you need to get the result of each future as soon as possible, - or if you need the result of some futures even if others produce - errors, you can use ``WaitIterator``:: - - wait_iterator = gen.WaitIterator(future1, future2) - while not wait_iterator.done(): - try: - result = yield wait_iterator.next() - except Exception as e: - print("Error {} from {}".format(e, wait_iterator.current_future)) - else: - print("Result {} received from {} at {}".format( - result, wait_iterator.current_future, - wait_iterator.current_index)) - - Because results are returned as soon as they are available the - output from the iterator *will not be in the same order as the - input arguments*. If you need to know which future produced the - current result, you can use the attributes - ``WaitIterator.current_future``, or ``WaitIterator.current_index`` - to get the index of the future from the input list. (if keyword - arguments were used in the construction of the `WaitIterator`, - ``current_index`` will use the corresponding keyword). - - On Python 3.5, `WaitIterator` implements the async iterator - protocol, so it can be used with the ``async for`` statement (note - that in this version the entire iteration is aborted if any value - raises an exception, while the previous example can continue past - individual errors):: - - async for result in gen.WaitIterator(future1, future2): - print("Result {} received from {} at {}".format( - result, wait_iterator.current_future, - wait_iterator.current_index)) - - .. versionadded:: 4.1 - - .. versionchanged:: 4.3 - Added ``async for`` support in Python 3.5. - - """ - def __init__(self, *args, **kwargs): - if args and kwargs: - raise ValueError( - "You must provide args or kwargs, not both") - - if kwargs: - self._unfinished = dict((f, k) for (k, f) in kwargs.items()) - futures = list(kwargs.values()) - else: - self._unfinished = dict((f, i) for (i, f) in enumerate(args)) - futures = args - - self._finished = collections.deque() - self.current_index = self.current_future = None - self._running_future = None - - for future in futures: - future.add_done_callback(self._done_callback) - - def done(self): - """Returns True if this iterator has no more results.""" - if self._finished or self._unfinished: - return False - # Clear the 'current' values when iteration is done. - self.current_index = self.current_future = None - return True - - def next(self): - """Returns a `.Future` that will yield the next available result. - - Note that this `.Future` will not be the same object as any of - the inputs. - """ - self._running_future = TracebackFuture() - - if self._finished: - self._return_result(self._finished.popleft()) - - return self._running_future - - def _done_callback(self, done): - if self._running_future and not self._running_future.done(): - self._return_result(done) - else: - self._finished.append(done) - - def _return_result(self, done): - """Called set the returned future's state that of the future - we yielded, and set the current future for the iterator. - """ - chain_future(done, self._running_future) - - self.current_future = done - self.current_index = self._unfinished.pop(done) - - def __aiter__(self): - raise self - - def __anext__(self): - if self.done(): - # Lookup by name to silence pyflakes on older versions. - raise getattr(builtins, 'StopAsyncIteration')() - return self.next() - - -class YieldPoint(object): - """Base class for objects that may be yielded from the generator. - - .. deprecated:: 4.0 - Use `Futures <.Future>` instead. - """ - def start(self, runner): - """Called by the runner after the generator has yielded. - - No other methods will be called on this object before ``start``. - """ - raise NotImplementedError() - - def is_ready(self): - """Called by the runner to determine whether to resume the generator. - - Returns a boolean; may be called more than once. - """ - raise NotImplementedError() - - def get_result(self): - """Returns the value to use as the result of the yield expression. - - This method will only be called once, and only after `is_ready` - has returned true. - """ - raise NotImplementedError() - - -class Callback(YieldPoint): - """Returns a callable object that will allow a matching `Wait` to proceed. - - The key may be any value suitable for use as a dictionary key, and is - used to match ``Callbacks`` to their corresponding ``Waits``. The key - must be unique among outstanding callbacks within a single run of the - generator function, but may be reused across different runs of the same - function (so constants generally work fine). - - The callback may be called with zero or one arguments; if an argument - is given it will be returned by `Wait`. - - .. deprecated:: 4.0 - Use `Futures <.Future>` instead. - """ - def __init__(self, key): - self.key = key - - def start(self, runner): - self.runner = runner - runner.register_callback(self.key) - - def is_ready(self): - return True - - def get_result(self): - return self.runner.result_callback(self.key) - - -class Wait(YieldPoint): - """Returns the argument passed to the result of a previous `Callback`. - - .. deprecated:: 4.0 - Use `Futures <.Future>` instead. - """ - def __init__(self, key): - self.key = key - - def start(self, runner): - self.runner = runner - - def is_ready(self): - return self.runner.is_ready(self.key) - - def get_result(self): - return self.runner.pop_result(self.key) - - -class WaitAll(YieldPoint): - """Returns the results of multiple previous `Callbacks `. - - The argument is a sequence of `Callback` keys, and the result is - a list of results in the same order. - - `WaitAll` is equivalent to yielding a list of `Wait` objects. - - .. deprecated:: 4.0 - Use `Futures <.Future>` instead. - """ - def __init__(self, keys): - self.keys = keys - - def start(self, runner): - self.runner = runner - - def is_ready(self): - return all(self.runner.is_ready(key) for key in self.keys) - - def get_result(self): - return [self.runner.pop_result(key) for key in self.keys] - - -def Task(func, *args, **kwargs): - """Adapts a callback-based asynchronous function for use in coroutines. - - Takes a function (and optional additional arguments) and runs it with - those arguments plus a ``callback`` keyword argument. The argument passed - to the callback is returned as the result of the yield expression. - - .. versionchanged:: 4.0 - ``gen.Task`` is now a function that returns a `.Future`, instead of - a subclass of `YieldPoint`. It still behaves the same way when - yielded. - """ - future = Future() - - def handle_exception(typ, value, tb): - if future.done(): - return False - future.set_exc_info((typ, value, tb)) - return True - - def set_result(result): - if future.done(): - return - future.set_result(result) - with stack_context.ExceptionStackContext(handle_exception): - func(*args, callback=_argument_adapter(set_result), **kwargs) - return future - - -class YieldFuture(YieldPoint): - def __init__(self, future, io_loop=None): - """Adapts a `.Future` to the `YieldPoint` interface. - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - """ - self.future = future - self.io_loop = io_loop or IOLoop.current() - - def start(self, runner): - if not self.future.done(): - self.runner = runner - self.key = object() - runner.register_callback(self.key) - self.io_loop.add_future(self.future, runner.result_callback(self.key)) - else: - self.runner = None - self.result_fn = self.future.result - - def is_ready(self): - if self.runner is not None: - return self.runner.is_ready(self.key) - else: - return True - - def get_result(self): - if self.runner is not None: - return self.runner.pop_result(self.key).result() - else: - return self.result_fn() - - -def _contains_yieldpoint(children): - """Returns True if ``children`` contains any YieldPoints. - - ``children`` may be a dict or a list, as used by `MultiYieldPoint` - and `multi_future`. - """ - if isinstance(children, dict): - return any(isinstance(i, YieldPoint) for i in children.values()) - if isinstance(children, list): - return any(isinstance(i, YieldPoint) for i in children) - return False - - -def multi(children, quiet_exceptions=()): - """Runs multiple asynchronous operations in parallel. - - ``children`` may either be a list or a dict whose values are - yieldable objects. ``multi()`` returns a new yieldable - object that resolves to a parallel structure containing their - results. If ``children`` is a list, the result is a list of - results in the same order; if it is a dict, the result is a dict - with the same keys. - - That is, ``results = yield multi(list_of_futures)`` is equivalent - to:: - - results = [] - for future in list_of_futures: - results.append(yield future) - - If any children raise exceptions, ``multi()`` will raise the first - one. All others will be logged, unless they are of types - contained in the ``quiet_exceptions`` argument. - - If any of the inputs are `YieldPoints `, the returned - yieldable object is a `YieldPoint`. Otherwise, returns a `.Future`. - This means that the result of `multi` can be used in a native - coroutine if and only if all of its children can be. - - In a ``yield``-based coroutine, it is not normally necessary to - call this function directly, since the coroutine runner will - do it automatically when a list or dict is yielded. However, - it is necessary in ``await``-based coroutines, or to pass - the ``quiet_exceptions`` argument. - - This function is available under the names ``multi()`` and ``Multi()`` - for historical reasons. - - .. versionchanged:: 4.2 - If multiple yieldables fail, any exceptions after the first - (which is raised) will be logged. Added the ``quiet_exceptions`` - argument to suppress this logging for selected exception types. - - .. versionchanged:: 4.3 - Replaced the class ``Multi`` and the function ``multi_future`` - with a unified function ``multi``. Added support for yieldables - other than `YieldPoint` and `.Future`. - - """ - if _contains_yieldpoint(children): - return MultiYieldPoint(children, quiet_exceptions=quiet_exceptions) - else: - return multi_future(children, quiet_exceptions=quiet_exceptions) - - -Multi = multi - - -class MultiYieldPoint(YieldPoint): - """Runs multiple asynchronous operations in parallel. - - This class is similar to `multi`, but it always creates a stack - context even when no children require it. It is not compatible with - native coroutines. - - .. versionchanged:: 4.2 - If multiple ``YieldPoints`` fail, any exceptions after the first - (which is raised) will be logged. Added the ``quiet_exceptions`` - argument to suppress this logging for selected exception types. - - .. versionchanged:: 4.3 - Renamed from ``Multi`` to ``MultiYieldPoint``. The name ``Multi`` - remains as an alias for the equivalent `multi` function. - - .. deprecated:: 4.3 - Use `multi` instead. - """ - def __init__(self, children, quiet_exceptions=()): - self.keys = None - if isinstance(children, dict): - self.keys = list(children.keys()) - children = children.values() - self.children = [] - for i in children: - if not isinstance(i, YieldPoint): - i = convert_yielded(i) - if is_future(i): - i = YieldFuture(i) - self.children.append(i) - assert all(isinstance(i, YieldPoint) for i in self.children) - self.unfinished_children = set(self.children) - self.quiet_exceptions = quiet_exceptions - - def start(self, runner): - for i in self.children: - i.start(runner) - - def is_ready(self): - finished = list(itertools.takewhile( - lambda i: i.is_ready(), self.unfinished_children)) - self.unfinished_children.difference_update(finished) - return not self.unfinished_children - - def get_result(self): - result_list = [] - exc_info = None - for f in self.children: - try: - result_list.append(f.get_result()) - except Exception as e: - if exc_info is None: - exc_info = sys.exc_info() - else: - if not isinstance(e, self.quiet_exceptions): - app_log.error("Multiple exceptions in yield list", - exc_info=True) - if exc_info is not None: - raise_exc_info(exc_info) - if self.keys is not None: - return dict(zip(self.keys, result_list)) - else: - return list(result_list) - - -def multi_future(children, quiet_exceptions=()): - """Wait for multiple asynchronous futures in parallel. - - This function is similar to `multi`, but does not support - `YieldPoints `. - - .. versionadded:: 4.0 - - .. versionchanged:: 4.2 - If multiple ``Futures`` fail, any exceptions after the first (which is - raised) will be logged. Added the ``quiet_exceptions`` - argument to suppress this logging for selected exception types. - - .. deprecated:: 4.3 - Use `multi` instead. - """ - if isinstance(children, dict): - keys = list(children.keys()) - children = children.values() - else: - keys = None - children = list(map(convert_yielded, children)) - assert all(is_future(i) for i in children) - unfinished_children = set(children) - - future = Future() - if not children: - future.set_result({} if keys is not None else []) - - def callback(f): - unfinished_children.remove(f) - if not unfinished_children: - result_list = [] - for f in children: - try: - result_list.append(f.result()) - except Exception as e: - if future.done(): - if not isinstance(e, quiet_exceptions): - app_log.error("Multiple exceptions in yield list", - exc_info=True) - else: - future.set_exc_info(sys.exc_info()) - if not future.done(): - if keys is not None: - future.set_result(dict(zip(keys, result_list))) - else: - future.set_result(result_list) - - listening = set() - for f in children: - if f not in listening: - listening.add(f) - f.add_done_callback(callback) - return future - - -def maybe_future(x): - """Converts ``x`` into a `.Future`. - - If ``x`` is already a `.Future`, it is simply returned; otherwise - it is wrapped in a new `.Future`. This is suitable for use as - ``result = yield gen.maybe_future(f())`` when you don't know whether - ``f()`` returns a `.Future` or not. - - .. deprecated:: 4.3 - This function only handles ``Futures``, not other yieldable objects. - Instead of `maybe_future`, check for the non-future result types - you expect (often just ``None``), and ``yield`` anything unknown. - """ - if is_future(x): - return x - else: - fut = Future() - fut.set_result(x) - return fut - - -def with_timeout(timeout, future, io_loop=None, quiet_exceptions=()): - """Wraps a `.Future` (or other yieldable object) in a timeout. - - Raises `TimeoutError` if the input future does not complete before - ``timeout``, which may be specified in any form allowed by - `.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time - relative to `.IOLoop.time`) - - If the wrapped `.Future` fails after it has timed out, the exception - will be logged unless it is of a type contained in ``quiet_exceptions`` - (which may be an exception type or a sequence of types). - - Does not support `YieldPoint` subclasses. - - .. versionadded:: 4.0 - - .. versionchanged:: 4.1 - Added the ``quiet_exceptions`` argument and the logging of unhandled - exceptions. - - .. versionchanged:: 4.4 - Added support for yieldable objects other than `.Future`. - """ - # TODO: allow YieldPoints in addition to other yieldables? - # Tricky to do with stack_context semantics. - # - # It's tempting to optimize this by cancelling the input future on timeout - # instead of creating a new one, but A) we can't know if we are the only - # one waiting on the input future, so cancelling it might disrupt other - # callers and B) concurrent futures can only be cancelled while they are - # in the queue, so cancellation cannot reliably bound our waiting time. - future = convert_yielded(future) - result = Future() - chain_future(future, result) - if io_loop is None: - io_loop = IOLoop.current() - - def error_callback(future): - try: - future.result() - except Exception as e: - if not isinstance(e, quiet_exceptions): - app_log.error("Exception in Future %r after timeout", - future, exc_info=True) - - def timeout_callback(): - result.set_exception(TimeoutError("Timeout")) - # In case the wrapped future goes on to fail, log it. - future.add_done_callback(error_callback) - timeout_handle = io_loop.add_timeout( - timeout, timeout_callback) - if isinstance(future, Future): - # We know this future will resolve on the IOLoop, so we don't - # need the extra thread-safety of IOLoop.add_future (and we also - # don't care about StackContext here. - future.add_done_callback( - lambda future: io_loop.remove_timeout(timeout_handle)) - else: - # concurrent.futures.Futures may resolve on any thread, so we - # need to route them back to the IOLoop. - io_loop.add_future( - future, lambda future: io_loop.remove_timeout(timeout_handle)) - return result - - -def sleep(duration): - """Return a `.Future` that resolves after the given number of seconds. - - When used with ``yield`` in a coroutine, this is a non-blocking - analogue to `time.sleep` (which should not be used in coroutines - because it is blocking):: - - yield gen.sleep(0.5) - - Note that calling this function on its own does nothing; you must - wait on the `.Future` it returns (usually by yielding it). - - .. versionadded:: 4.1 - """ - f = Future() - IOLoop.current().call_later(duration, lambda: f.set_result(None)) - return f - - -_null_future = Future() -_null_future.set_result(None) - -moment = Future() -moment.__doc__ = \ - """A special object which may be yielded to allow the IOLoop to run for -one iteration. - -This is not needed in normal use but it can be helpful in long-running -coroutines that are likely to yield Futures that are ready instantly. - -Usage: ``yield gen.moment`` - -.. versionadded:: 4.0 - -.. deprecated:: 4.5 - ``yield None`` is now equivalent to ``yield gen.moment``. -""" -moment.set_result(None) - - -class Runner(object): - """Internal implementation of `tornado.gen.engine`. - - Maintains information about pending callbacks and their results. - - The results of the generator are stored in ``result_future`` (a - `.TracebackFuture`) - """ - def __init__(self, gen, result_future, first_yielded): - self.gen = gen - self.result_future = result_future - self.future = _null_future - self.yield_point = None - self.pending_callbacks = None - self.results = None - self.running = False - self.finished = False - self.had_exception = False - self.io_loop = IOLoop.current() - # For efficiency, we do not create a stack context until we - # reach a YieldPoint (stack contexts are required for the historical - # semantics of YieldPoints, but not for Futures). When we have - # done so, this field will be set and must be called at the end - # of the coroutine. - self.stack_context_deactivate = None - if self.handle_yield(first_yielded): - gen = result_future = first_yielded = None - self.run() - - def register_callback(self, key): - """Adds ``key`` to the list of callbacks.""" - if self.pending_callbacks is None: - # Lazily initialize the old-style YieldPoint data structures. - self.pending_callbacks = set() - self.results = {} - if key in self.pending_callbacks: - raise KeyReuseError("key %r is already pending" % (key,)) - self.pending_callbacks.add(key) - - def is_ready(self, key): - """Returns true if a result is available for ``key``.""" - if self.pending_callbacks is None or key not in self.pending_callbacks: - raise UnknownKeyError("key %r is not pending" % (key,)) - return key in self.results - - def set_result(self, key, result): - """Sets the result for ``key`` and attempts to resume the generator.""" - self.results[key] = result - if self.yield_point is not None and self.yield_point.is_ready(): - try: - self.future.set_result(self.yield_point.get_result()) - except: - self.future.set_exc_info(sys.exc_info()) - self.yield_point = None - self.run() - - def pop_result(self, key): - """Returns the result for ``key`` and unregisters it.""" - self.pending_callbacks.remove(key) - return self.results.pop(key) - - def run(self): - """Starts or resumes the generator, running until it reaches a - yield point that is not ready. - """ - if self.running or self.finished: - return - try: - self.running = True - while True: - future = self.future - if not future.done(): - return - self.future = None - try: - orig_stack_contexts = stack_context._state.contexts - exc_info = None - - try: - value = future.result() - except Exception: - self.had_exception = True - exc_info = sys.exc_info() - future = None - - if exc_info is not None: - try: - yielded = self.gen.throw(*exc_info) - finally: - # Break up a reference to itself - # for faster GC on CPython. - exc_info = None - else: - yielded = self.gen.send(value) - - if stack_context._state.contexts is not orig_stack_contexts: - self.gen.throw( - stack_context.StackContextInconsistentError( - 'stack_context inconsistency (probably caused ' - 'by yield within a "with StackContext" block)')) - except (StopIteration, Return) as e: - self.finished = True - self.future = _null_future - if self.pending_callbacks and not self.had_exception: - # If we ran cleanly without waiting on all callbacks - # raise an error (really more of a warning). If we - # had an exception then some callbacks may have been - # orphaned, so skip the check in that case. - raise LeakedCallbackError( - "finished without waiting for callbacks %r" % - self.pending_callbacks) - self.result_future.set_result(_value_from_stopiteration(e)) - self.result_future = None - self._deactivate_stack_context() - return - except Exception: - self.finished = True - self.future = _null_future - self.result_future.set_exc_info(sys.exc_info()) - self.result_future = None - self._deactivate_stack_context() - return - if not self.handle_yield(yielded): - return - yielded = None - finally: - self.running = False - - def handle_yield(self, yielded): - # Lists containing YieldPoints require stack contexts; - # other lists are handled in convert_yielded. - if _contains_yieldpoint(yielded): - yielded = multi(yielded) - - if isinstance(yielded, YieldPoint): - # YieldPoints are too closely coupled to the Runner to go - # through the generic convert_yielded mechanism. - self.future = TracebackFuture() - - def start_yield_point(): - try: - yielded.start(self) - if yielded.is_ready(): - self.future.set_result( - yielded.get_result()) - else: - self.yield_point = yielded - except Exception: - self.future = TracebackFuture() - self.future.set_exc_info(sys.exc_info()) - - if self.stack_context_deactivate is None: - # Start a stack context if this is the first - # YieldPoint we've seen. - with stack_context.ExceptionStackContext( - self.handle_exception) as deactivate: - self.stack_context_deactivate = deactivate - - def cb(): - start_yield_point() - self.run() - self.io_loop.add_callback(cb) - return False - else: - start_yield_point() - else: - try: - self.future = convert_yielded(yielded) - except BadYieldError: - self.future = TracebackFuture() - self.future.set_exc_info(sys.exc_info()) - - if not self.future.done() or self.future is moment: - def inner(f): - # Break a reference cycle to speed GC. - f = None # noqa - self.run() - self.io_loop.add_future( - self.future, inner) - return False - return True - - def result_callback(self, key): - return stack_context.wrap(_argument_adapter( - functools.partial(self.set_result, key))) - - def handle_exception(self, typ, value, tb): - if not self.running and not self.finished: - self.future = TracebackFuture() - self.future.set_exc_info((typ, value, tb)) - self.run() - return True - else: - return False - - def _deactivate_stack_context(self): - if self.stack_context_deactivate is not None: - self.stack_context_deactivate() - self.stack_context_deactivate = None - - -Arguments = collections.namedtuple('Arguments', ['args', 'kwargs']) - - -def _argument_adapter(callback): - """Returns a function that when invoked runs ``callback`` with one arg. - - If the function returned by this function is called with exactly - one argument, that argument is passed to ``callback``. Otherwise - the args tuple and kwargs dict are wrapped in an `Arguments` object. - """ - def wrapper(*args, **kwargs): - if kwargs or len(args) > 1: - callback(Arguments(args, kwargs)) - elif args: - callback(args[0]) - else: - callback(None) - return wrapper - - -# Convert Awaitables into Futures. It is unfortunately possible -# to have infinite recursion here if those Awaitables assume that -# we're using a different coroutine runner and yield objects -# we don't understand. If that happens, the solution is to -# register that runner's yieldable objects with convert_yielded. -if sys.version_info >= (3, 3): - exec(textwrap.dedent(""" - @coroutine - def _wrap_awaitable(x): - if hasattr(x, '__await__'): - x = x.__await__() - return (yield from x) - """)) -else: - # Py2-compatible version for use with Cython. - # Copied from PEP 380. - @coroutine - def _wrap_awaitable(x): - if hasattr(x, '__await__'): - _i = x.__await__() - else: - _i = iter(x) - try: - _y = next(_i) - except StopIteration as _e: - _r = _value_from_stopiteration(_e) - else: - while 1: - try: - _s = yield _y - except GeneratorExit as _e: - try: - _m = _i.close - except AttributeError: - pass - else: - _m() - raise _e - except BaseException as _e: - _x = sys.exc_info() - try: - _m = _i.throw - except AttributeError: - raise _e - else: - try: - _y = _m(*_x) - except StopIteration as _e: - _r = _value_from_stopiteration(_e) - break - else: - try: - if _s is None: - _y = next(_i) - else: - _y = _i.send(_s) - except StopIteration as _e: - _r = _value_from_stopiteration(_e) - break - raise Return(_r) - - -def convert_yielded(yielded): - """Convert a yielded object into a `.Future`. - - The default implementation accepts lists, dictionaries, and Futures. - - If the `~functools.singledispatch` library is available, this function - may be extended to support additional types. For example:: - - @convert_yielded.register(asyncio.Future) - def _(asyncio_future): - return tornado.platform.asyncio.to_tornado_future(asyncio_future) - - .. versionadded:: 4.1 - """ - # Lists and dicts containing YieldPoints were handled earlier. - if yielded is None: - return moment - elif isinstance(yielded, (list, dict)): - return multi(yielded) - elif is_future(yielded): - return yielded - elif isawaitable(yielded): - return _wrap_awaitable(yielded) - else: - raise BadYieldError("yielded unknown object %r" % (yielded,)) - - -if singledispatch is not None: - convert_yielded = singledispatch(convert_yielded) - - try: - # If we can import t.p.asyncio, do it for its side effect - # (registering asyncio.Future with convert_yielded). - # It's ugly to do this here, but it prevents a cryptic - # infinite recursion in _wrap_awaitable. - # Note that even with this, asyncio integration is unlikely - # to work unless the application also configures AsyncIOLoop, - # but at least the error messages in that case are more - # comprehensible than a stack overflow. - import salt.ext.tornado.platform.asyncio - except ImportError: - pass - else: - # Reference the imported module to make pyflakes happy. - tornado diff --git a/salt/ext/tornado/http1connection.py b/salt/ext/tornado/http1connection.py deleted file mode 100644 index 7ed1078faa7..00000000000 --- a/salt/ext/tornado/http1connection.py +++ /dev/null @@ -1,743 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2014 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Client and server implementations of HTTP/1.x. - -.. versionadded:: 4.0 -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import re - -from salt.ext.tornado.concurrent import Future -from salt.ext.tornado.escape import native_str, utf8 -from salt.ext.tornado import gen -from salt.ext.tornado import httputil -from salt.ext.tornado import iostream -from salt.ext.tornado.log import gen_log, app_log -from salt.ext.tornado import stack_context -from salt.ext.tornado.util import GzipDecompressor, PY3 - - -class _QuietException(Exception): - def __init__(self): - pass - - -class _ExceptionLoggingContext(object): - """Used with the ``with`` statement when calling delegate methods to - log any exceptions with the given logger. Any exceptions caught are - converted to _QuietException - """ - def __init__(self, logger): - self.logger = logger - - def __enter__(self): - pass - - def __exit__(self, typ, value, tb): - if value is not None: - self.logger.error("Uncaught exception", exc_info=(typ, value, tb)) - raise _QuietException - - -class HTTP1ConnectionParameters(object): - """Parameters for `.HTTP1Connection` and `.HTTP1ServerConnection`. - """ - def __init__(self, no_keep_alive=False, chunk_size=None, - max_header_size=None, header_timeout=None, max_body_size=None, - body_timeout=None, decompress=False): - """ - :arg bool no_keep_alive: If true, always close the connection after - one request. - :arg int chunk_size: how much data to read into memory at once - :arg int max_header_size: maximum amount of data for HTTP headers - :arg float header_timeout: how long to wait for all headers (seconds) - :arg int max_body_size: maximum amount of data for body - :arg float body_timeout: how long to wait while reading body (seconds) - :arg bool decompress: if true, decode incoming - ``Content-Encoding: gzip`` - """ - self.no_keep_alive = no_keep_alive - self.chunk_size = chunk_size or 65536 - self.max_header_size = max_header_size or 65536 - self.header_timeout = header_timeout - self.max_body_size = max_body_size - self.body_timeout = body_timeout - self.decompress = decompress - - -class HTTP1Connection(httputil.HTTPConnection): - """Implements the HTTP/1.x protocol. - - This class can be on its own for clients, or via `HTTP1ServerConnection` - for servers. - """ - def __init__(self, stream, is_client, params=None, context=None): - """ - :arg stream: an `.IOStream` - :arg bool is_client: client or server - :arg params: a `.HTTP1ConnectionParameters` instance or ``None`` - :arg context: an opaque application-defined object that can be accessed - as ``connection.context``. - """ - self.is_client = is_client - self.stream = stream - if params is None: - params = HTTP1ConnectionParameters() - self.params = params - self.context = context - self.no_keep_alive = params.no_keep_alive - # The body limits can be altered by the delegate, so save them - # here instead of just referencing self.params later. - self._max_body_size = (self.params.max_body_size or - self.stream.max_buffer_size) - self._body_timeout = self.params.body_timeout - # _write_finished is set to True when finish() has been called, - # i.e. there will be no more data sent. Data may still be in the - # stream's write buffer. - self._write_finished = False - # True when we have read the entire incoming body. - self._read_finished = False - # _finish_future resolves when all data has been written and flushed - # to the IOStream. - self._finish_future = Future() - # If true, the connection should be closed after this request - # (after the response has been written in the server side, - # and after it has been read in the client) - self._disconnect_on_finish = False - self._clear_callbacks() - # Save the start lines after we read or write them; they - # affect later processing (e.g. 304 responses and HEAD methods - # have content-length but no bodies) - self._request_start_line = None - self._response_start_line = None - self._request_headers = None - # True if we are writing output with chunked encoding. - self._chunking_output = None - # While reading a body with a content-length, this is the - # amount left to read. - self._expected_content_remaining = None - # A Future for our outgoing writes, returned by IOStream.write. - self._pending_write = None - - def read_response(self, delegate): - """Read a single HTTP response. - - Typical client-mode usage is to write a request using `write_headers`, - `write`, and `finish`, and then call ``read_response``. - - :arg delegate: a `.HTTPMessageDelegate` - - Returns a `.Future` that resolves to None after the full response has - been read. - """ - if self.params.decompress: - delegate = _GzipMessageDelegate(delegate, self.params.chunk_size) - return self._read_message(delegate) - - @gen.coroutine - def _read_message(self, delegate): - need_delegate_close = False - try: - header_future = self.stream.read_until_regex( - b"\r?\n\r?\n", - max_bytes=self.params.max_header_size) - if self.params.header_timeout is None: - header_data = yield header_future - else: - try: - header_data = yield gen.with_timeout( - self.stream.io_loop.time() + self.params.header_timeout, - header_future, - io_loop=self.stream.io_loop, - quiet_exceptions=iostream.StreamClosedError) - except gen.TimeoutError: - self.close() - raise gen.Return(False) - start_line, headers = self._parse_headers(header_data) - if self.is_client: - start_line = httputil.parse_response_start_line(start_line) - self._response_start_line = start_line - else: - start_line = httputil.parse_request_start_line(start_line) - self._request_start_line = start_line - self._request_headers = headers - - self._disconnect_on_finish = not self._can_keep_alive( - start_line, headers) - need_delegate_close = True - with _ExceptionLoggingContext(app_log): - header_future = delegate.headers_received(start_line, headers) - if header_future is not None: - yield header_future - if self.stream is None: - # We've been detached. - need_delegate_close = False - raise gen.Return(False) - skip_body = False - if self.is_client: - if (self._request_start_line is not None and - self._request_start_line.method == 'HEAD'): - skip_body = True - code = start_line.code - if code == 304: - # 304 responses may include the content-length header - # but do not actually have a body. - # http://tools.ietf.org/html/rfc7230#section-3.3 - skip_body = True - if code >= 100 and code < 200: - # 1xx responses should never indicate the presence of - # a body. - if ('Content-Length' in headers or - 'Transfer-Encoding' in headers): - raise httputil.HTTPInputError( - "Response code %d cannot have body" % code) - # TODO: client delegates will get headers_received twice - # in the case of a 100-continue. Document or change? - yield self._read_message(delegate) - else: - if (headers.get("Expect") == "100-continue" and - not self._write_finished): - self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") - if not skip_body: - body_future = self._read_body( - start_line.code if self.is_client else 0, headers, delegate) - if body_future is not None: - if self._body_timeout is None: - yield body_future - else: - try: - yield gen.with_timeout( - self.stream.io_loop.time() + self._body_timeout, - body_future, self.stream.io_loop, - quiet_exceptions=iostream.StreamClosedError) - except gen.TimeoutError: - gen_log.info("Timeout reading body from %s", - self.context) - self.stream.close() - raise gen.Return(False) - self._read_finished = True - if not self._write_finished or self.is_client: - need_delegate_close = False - with _ExceptionLoggingContext(app_log): - delegate.finish() - # If we're waiting for the application to produce an asynchronous - # response, and we're not detached, register a close callback - # on the stream (we didn't need one while we were reading) - if (not self._finish_future.done() and - self.stream is not None and - not self.stream.closed()): - self.stream.set_close_callback(self._on_connection_close) - yield self._finish_future - if self.is_client and self._disconnect_on_finish: - self.close() - if self.stream is None: - raise gen.Return(False) - except httputil.HTTPInputError as e: - gen_log.info("Malformed HTTP message from %s: %s", - self.context, e) - self.close() - raise gen.Return(False) - finally: - if need_delegate_close: - with _ExceptionLoggingContext(app_log): - delegate.on_connection_close() - header_future = None - self._clear_callbacks() - raise gen.Return(True) - - def _clear_callbacks(self): - """Clears the callback attributes. - - This allows the request handler to be garbage collected more - quickly in CPython by breaking up reference cycles. - """ - self._write_callback = None - self._write_future = None - self._close_callback = None - if self.stream is not None: - self.stream.set_close_callback(None) - - def set_close_callback(self, callback): - """Sets a callback that will be run when the connection is closed. - - .. deprecated:: 4.0 - Use `.HTTPMessageDelegate.on_connection_close` instead. - """ - self._close_callback = stack_context.wrap(callback) - - def _on_connection_close(self): - # Note that this callback is only registered on the IOStream - # when we have finished reading the request and are waiting for - # the application to produce its response. - if self._close_callback is not None: - callback = self._close_callback - self._close_callback = None - callback() - if not self._finish_future.done(): - self._finish_future.set_result(None) - self._clear_callbacks() - - def close(self): - if self.stream is not None: - self.stream.close() - self._clear_callbacks() - if not self._finish_future.done(): - self._finish_future.set_result(None) - - def detach(self): - """Take control of the underlying stream. - - Returns the underlying `.IOStream` object and stops all further - HTTP processing. May only be called during - `.HTTPMessageDelegate.headers_received`. Intended for implementing - protocols like websockets that tunnel over an HTTP handshake. - """ - self._clear_callbacks() - stream = self.stream - self.stream = None - if not self._finish_future.done(): - self._finish_future.set_result(None) - return stream - - def set_body_timeout(self, timeout): - """Sets the body timeout for a single request. - - Overrides the value from `.HTTP1ConnectionParameters`. - """ - self._body_timeout = timeout - - def set_max_body_size(self, max_body_size): - """Sets the body size limit for a single request. - - Overrides the value from `.HTTP1ConnectionParameters`. - """ - self._max_body_size = max_body_size - - def write_headers(self, start_line, headers, chunk=None, callback=None): - """Implements `.HTTPConnection.write_headers`.""" - lines = [] - if self.is_client: - self._request_start_line = start_line - lines.append(utf8('%s %s HTTP/1.1' % (start_line[0], start_line[1]))) - # Client requests with a non-empty body must have either a - # Content-Length or a Transfer-Encoding. - self._chunking_output = ( - start_line.method in ('POST', 'PUT', 'PATCH') and - 'Content-Length' not in headers and - 'Transfer-Encoding' not in headers) - else: - self._response_start_line = start_line - lines.append(utf8('HTTP/1.1 %d %s' % (start_line[1], start_line[2]))) - self._chunking_output = ( - # TODO: should this use - # self._request_start_line.version or - # start_line.version? - self._request_start_line.version == 'HTTP/1.1' and - # 1xx, 204 and 304 responses have no body (not even a zero-length - # body), and so should not have either Content-Length or - # Transfer-Encoding headers. - start_line.code not in (204, 304) and - (start_line.code < 100 or start_line.code >= 200) and - # No need to chunk the output if a Content-Length is specified. - 'Content-Length' not in headers and - # Applications are discouraged from touching Transfer-Encoding, - # but if they do, leave it alone. - 'Transfer-Encoding' not in headers) - # If a 1.0 client asked for keep-alive, add the header. - if (self._request_start_line.version == 'HTTP/1.0' and - (self._request_headers.get('Connection', '').lower() == - 'keep-alive')): - headers['Connection'] = 'Keep-Alive' - if self._chunking_output: - headers['Transfer-Encoding'] = 'chunked' - if (not self.is_client and - (self._request_start_line.method == 'HEAD' or - start_line.code == 304)): - self._expected_content_remaining = 0 - elif 'Content-Length' in headers: - self._expected_content_remaining = int(headers['Content-Length']) - else: - self._expected_content_remaining = None - # TODO: headers are supposed to be of type str, but we still have some - # cases that let bytes slip through. Remove these native_str calls when those - # are fixed. - header_lines = (native_str(n) + ": " + native_str(v) for n, v in headers.get_all()) - if PY3: - lines.extend(l.encode('latin1') for l in header_lines) - else: - lines.extend(header_lines) - for line in lines: - if b'\n' in line: - raise ValueError('Newline in header: ' + repr(line)) - future = None - if self.stream.closed(): - future = self._write_future = Future() - future.set_exception(iostream.StreamClosedError()) - future.exception() - else: - if callback is not None: - self._write_callback = stack_context.wrap(callback) - else: - future = self._write_future = Future() - data = b"\r\n".join(lines) + b"\r\n\r\n" - if chunk: - data += self._format_chunk(chunk) - self._pending_write = self.stream.write(data) - self._pending_write.add_done_callback(self._on_write_complete) - return future - - def _format_chunk(self, chunk): - if self._expected_content_remaining is not None: - self._expected_content_remaining -= len(chunk) - if self._expected_content_remaining < 0: - # Close the stream now to stop further framing errors. - self.stream.close() - raise httputil.HTTPOutputError( - "Tried to write more data than Content-Length") - if self._chunking_output and chunk: - # Don't write out empty chunks because that means END-OF-STREAM - # with chunked encoding - return utf8("%x" % len(chunk)) + b"\r\n" + chunk + b"\r\n" - else: - return chunk - - def write(self, chunk, callback=None): - """Implements `.HTTPConnection.write`. - - For backwards compatibility is is allowed but deprecated to - skip `write_headers` and instead call `write()` with a - pre-encoded header block. - """ - future = None - if self.stream.closed(): - future = self._write_future = Future() - self._write_future.set_exception(iostream.StreamClosedError()) - self._write_future.exception() - else: - if callback is not None: - self._write_callback = stack_context.wrap(callback) - else: - future = self._write_future = Future() - self._pending_write = self.stream.write(self._format_chunk(chunk)) - self._pending_write.add_done_callback(self._on_write_complete) - return future - - def finish(self): - """Implements `.HTTPConnection.finish`.""" - if (self._expected_content_remaining is not None and - self._expected_content_remaining != 0 and - not self.stream.closed()): - self.stream.close() - raise httputil.HTTPOutputError( - "Tried to write %d bytes less than Content-Length" % - self._expected_content_remaining) - if self._chunking_output: - if not self.stream.closed(): - self._pending_write = self.stream.write(b"0\r\n\r\n") - self._pending_write.add_done_callback(self._on_write_complete) - self._write_finished = True - # If the app finished the request while we're still reading, - # divert any remaining data away from the delegate and - # close the connection when we're done sending our response. - # Closing the connection is the only way to avoid reading the - # whole input body. - if not self._read_finished: - self._disconnect_on_finish = True - # No more data is coming, so instruct TCP to send any remaining - # data immediately instead of waiting for a full packet or ack. - self.stream.set_nodelay(True) - if self._pending_write is None: - self._finish_request(None) - else: - self._pending_write.add_done_callback(self._finish_request) - - def _on_write_complete(self, future): - exc = future.exception() - if exc is not None and not isinstance(exc, iostream.StreamClosedError): - future.result() - if self._write_callback is not None: - callback = self._write_callback - self._write_callback = None - self.stream.io_loop.add_callback(callback) - if self._write_future is not None: - future = self._write_future - self._write_future = None - future.set_result(None) - - def _can_keep_alive(self, start_line, headers): - if self.params.no_keep_alive: - return False - connection_header = headers.get("Connection") - if connection_header is not None: - connection_header = connection_header.lower() - if start_line.version == "HTTP/1.1": - return connection_header != "close" - elif ("Content-Length" in headers or - headers.get("Transfer-Encoding", "").lower() == "chunked" or - getattr(start_line, 'method', None) in ("HEAD", "GET")): - # start_line may be a request or response start line; only - # the former has a method attribute. - return connection_header == "keep-alive" - return False - - def _finish_request(self, future): - self._clear_callbacks() - if not self.is_client and self._disconnect_on_finish: - self.close() - return - # Turn Nagle's algorithm back on, leaving the stream in its - # default state for the next request. - self.stream.set_nodelay(False) - if not self._finish_future.done(): - self._finish_future.set_result(None) - - def _parse_headers(self, data): - # The lstrip removes newlines that some implementations sometimes - # insert between messages of a reused connection. Per RFC 7230, - # we SHOULD ignore at least one empty line before the request. - # http://tools.ietf.org/html/rfc7230#section-3.5 - data = native_str(data.decode('latin1')).lstrip("\r\n") - # RFC 7230 section allows for both CRLF and bare LF. - eol = data.find("\n") - start_line = data[:eol].rstrip("\r") - try: - headers = httputil.HTTPHeaders.parse(data[eol:]) - except ValueError: - # probably form split() if there was no ':' in the line - raise httputil.HTTPInputError("Malformed HTTP headers: %r" % - data[eol:100]) - return start_line, headers - - def _read_body(self, code, headers, delegate): - if "Content-Length" in headers: - if "Transfer-Encoding" in headers: - # Response cannot contain both Content-Length and - # Transfer-Encoding headers. - # http://tools.ietf.org/html/rfc7230#section-3.3.3 - raise httputil.HTTPInputError( - "Response with both Transfer-Encoding and Content-Length") - if "," in headers["Content-Length"]: - # Proxies sometimes cause Content-Length headers to get - # duplicated. If all the values are identical then we can - # use them but if they differ it's an error. - pieces = re.split(r',\s*', headers["Content-Length"]) - if any(i != pieces[0] for i in pieces): - raise httputil.HTTPInputError( - "Multiple unequal Content-Lengths: %r" % - headers["Content-Length"]) - headers["Content-Length"] = pieces[0] - - try: - content_length = int(headers["Content-Length"]) - except ValueError: - # Handles non-integer Content-Length value. - raise httputil.HTTPInputError( - "Only integer Content-Length is allowed: %s" % headers["Content-Length"]) - - if content_length > self._max_body_size: - raise httputil.HTTPInputError("Content-Length too long") - else: - content_length = None - - if code == 204: - # This response code is not allowed to have a non-empty body, - # and has an implicit length of zero instead of read-until-close. - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3 - if ("Transfer-Encoding" in headers or - content_length not in (None, 0)): - raise httputil.HTTPInputError( - "Response with code %d should not have body" % code) - content_length = 0 - - if content_length is not None: - return self._read_fixed_body(content_length, delegate) - if headers.get("Transfer-Encoding", "").lower() == "chunked": - return self._read_chunked_body(delegate) - if self.is_client: - return self._read_body_until_close(delegate) - return None - - @gen.coroutine - def _read_fixed_body(self, content_length, delegate): - while content_length > 0: - body = yield self.stream.read_bytes( - min(self.params.chunk_size, content_length), partial=True) - content_length -= len(body) - if not self._write_finished or self.is_client: - with _ExceptionLoggingContext(app_log): - ret = delegate.data_received(body) - if ret is not None: - yield ret - - @gen.coroutine - def _read_chunked_body(self, delegate): - # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1 - total_size = 0 - while True: - chunk_len = yield self.stream.read_until(b"\r\n", max_bytes=64) - chunk_len = int(chunk_len.strip(), 16) - if chunk_len == 0: - crlf = yield self.stream.read_bytes(2) - if crlf != b'\r\n': - raise httputil.HTTPInputError("improperly terminated chunked request") - return - total_size += chunk_len - if total_size > self._max_body_size: - raise httputil.HTTPInputError("chunked body too large") - bytes_to_read = chunk_len - while bytes_to_read: - chunk = yield self.stream.read_bytes( - min(bytes_to_read, self.params.chunk_size), partial=True) - bytes_to_read -= len(chunk) - if not self._write_finished or self.is_client: - with _ExceptionLoggingContext(app_log): - ret = delegate.data_received(chunk) - if ret is not None: - yield ret - # chunk ends with \r\n - crlf = yield self.stream.read_bytes(2) - assert crlf == b"\r\n" - - @gen.coroutine - def _read_body_until_close(self, delegate): - body = yield self.stream.read_until_close() - if not self._write_finished or self.is_client: - with _ExceptionLoggingContext(app_log): - delegate.data_received(body) - - -class _GzipMessageDelegate(httputil.HTTPMessageDelegate): - """Wraps an `HTTPMessageDelegate` to decode ``Content-Encoding: gzip``. - """ - def __init__(self, delegate, chunk_size): - self._delegate = delegate - self._chunk_size = chunk_size - self._decompressor = None - - def headers_received(self, start_line, headers): - if headers.get("Content-Encoding") == "gzip": - self._decompressor = GzipDecompressor() - # Downstream delegates will only see uncompressed data, - # so rename the content-encoding header. - # (but note that curl_httpclient doesn't do this). - headers.add("X-Consumed-Content-Encoding", - headers["Content-Encoding"]) - del headers["Content-Encoding"] - return self._delegate.headers_received(start_line, headers) - - @gen.coroutine - def data_received(self, chunk): - if self._decompressor: - compressed_data = chunk - while compressed_data: - decompressed = self._decompressor.decompress( - compressed_data, self._chunk_size) - if decompressed: - ret = self._delegate.data_received(decompressed) - if ret is not None: - yield ret - compressed_data = self._decompressor.unconsumed_tail - else: - ret = self._delegate.data_received(chunk) - if ret is not None: - yield ret - - def finish(self): - if self._decompressor is not None: - tail = self._decompressor.flush() - if tail: - # I believe the tail will always be empty (i.e. - # decompress will return all it can). The purpose - # of the flush call is to detect errors such - # as truncated input. But in case it ever returns - # anything, treat it as an extra chunk - self._delegate.data_received(tail) - return self._delegate.finish() - - def on_connection_close(self): - return self._delegate.on_connection_close() - - -class HTTP1ServerConnection(object): - """An HTTP/1.x server.""" - def __init__(self, stream, params=None, context=None): - """ - :arg stream: an `.IOStream` - :arg params: a `.HTTP1ConnectionParameters` or None - :arg context: an opaque application-defined object that is accessible - as ``connection.context`` - """ - self.stream = stream - if params is None: - params = HTTP1ConnectionParameters() - self.params = params - self.context = context - self._serving_future = None - - @gen.coroutine - def close(self): - """Closes the connection. - - Returns a `.Future` that resolves after the serving loop has exited. - """ - self.stream.close() - # Block until the serving loop is done, but ignore any exceptions - # (start_serving is already responsible for logging them). - try: - yield self._serving_future - except Exception: - pass - - def start_serving(self, delegate): - """Starts serving requests on this connection. - - :arg delegate: a `.HTTPServerConnectionDelegate` - """ - assert isinstance(delegate, httputil.HTTPServerConnectionDelegate) - self._serving_future = self._server_request_loop(delegate) - # Register the future on the IOLoop so its errors get logged. - self.stream.io_loop.add_future(self._serving_future, - lambda f: f.result()) - - @gen.coroutine - def _server_request_loop(self, delegate): - try: - while True: - conn = HTTP1Connection(self.stream, False, - self.params, self.context) - request_delegate = delegate.start_request(self, conn) - try: - ret = yield conn.read_response(request_delegate) - except (iostream.StreamClosedError, - iostream.UnsatisfiableReadError): - return - except _QuietException: - # This exception was already logged. - conn.close() - return - except Exception: - gen_log.error("Uncaught exception", exc_info=True) - conn.close() - return - if not ret: - return - yield gen.moment - finally: - delegate.on_close(self) diff --git a/salt/ext/tornado/httpclient.py b/salt/ext/tornado/httpclient.py deleted file mode 100644 index 20a12b2f152..00000000000 --- a/salt/ext/tornado/httpclient.py +++ /dev/null @@ -1,679 +0,0 @@ -"""Blocking and non-blocking HTTP client interfaces. - -This module defines a common interface shared by two implementations, -``simple_httpclient`` and ``curl_httpclient``. Applications may either -instantiate their chosen implementation class directly or use the -`AsyncHTTPClient` class from this module, which selects an implementation -that can be overridden with the `AsyncHTTPClient.configure` method. - -The default implementation is ``simple_httpclient``, and this is expected -to be suitable for most users' needs. However, some applications may wish -to switch to ``curl_httpclient`` for reasons such as the following: - -* ``curl_httpclient`` has some features not found in ``simple_httpclient``, - including support for HTTP proxies and the ability to use a specified - network interface. - -* ``curl_httpclient`` is more likely to be compatible with sites that are - not-quite-compliant with the HTTP spec, or sites that use little-exercised - features of HTTP. - -* ``curl_httpclient`` is faster. - -* ``curl_httpclient`` was the default prior to Tornado 2.0. - -Note that if you are using ``curl_httpclient``, it is highly -recommended that you use a recent version of ``libcurl`` and -``pycurl``. Currently the minimum supported version of libcurl is -7.22.0, and the minimum version of pycurl is 7.18.2. It is highly -recommended that your ``libcurl`` installation is built with -asynchronous DNS resolver (threaded or c-ares), otherwise you may -encounter various problems with request timeouts (for more -information, see -http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUTMS -and comments in curl_httpclient.py). - -To select ``curl_httpclient``, call `AsyncHTTPClient.configure` at startup:: - - AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import functools -import time -import weakref - -from salt.ext.tornado.concurrent import TracebackFuture -from salt.ext.tornado.escape import utf8, native_str -from salt.ext.tornado import httputil, stack_context -from salt.ext.tornado.ioloop import IOLoop -from salt.ext.tornado.util import Configurable - - -class HTTPClient(object): - """A blocking HTTP client. - - This interface is provided for convenience and testing; most applications - that are running an IOLoop will want to use `AsyncHTTPClient` instead. - Typical usage looks like this:: - - http_client = httpclient.HTTPClient() - try: - response = http_client.fetch("http://www.google.com/") - print(response.body) - except httpclient.HTTPError as e: - # HTTPError is raised for non-200 responses; the response - # can be found in e.response. - print("Error: " + str(e)) - except Exception as e: - # Other errors are possible, such as IOError. - print("Error: " + str(e)) - http_client.close() - """ - def __init__(self, async_client_class=None, **kwargs): - self._io_loop = IOLoop(make_current=False) - if async_client_class is None: - async_client_class = AsyncHTTPClient - self._async_client = async_client_class(self._io_loop, **kwargs) - self._closed = False - - def __del__(self): - self.close() - - def close(self): - """Closes the HTTPClient, freeing any resources used.""" - if not self._closed: - self._async_client.close() - self._io_loop.close() - self._closed = True - - def fetch(self, request, **kwargs): - """Executes a request, returning an `HTTPResponse`. - - The request may be either a string URL or an `HTTPRequest` object. - If it is a string, we construct an `HTTPRequest` using any additional - kwargs: ``HTTPRequest(request, **kwargs)`` - - If an error occurs during the fetch, we raise an `HTTPError` unless - the ``raise_error`` keyword argument is set to False. - """ - response = self._io_loop.run_sync(functools.partial( - self._async_client.fetch, request, **kwargs)) - return response - - -class AsyncHTTPClient(Configurable): - """An non-blocking HTTP client. - - Example usage:: - - def handle_response(response): - if response.error: - print("Error: %s" % response.error) - else: - print(response.body) - - http_client = AsyncHTTPClient() - http_client.fetch("http://www.google.com/", handle_response) - - The constructor for this class is magic in several respects: It - actually creates an instance of an implementation-specific - subclass, and instances are reused as a kind of pseudo-singleton - (one per `.IOLoop`). The keyword argument ``force_instance=True`` - can be used to suppress this singleton behavior. Unless - ``force_instance=True`` is used, no arguments other than - ``io_loop`` should be passed to the `AsyncHTTPClient` constructor. - The implementation subclass as well as arguments to its - constructor can be set with the static method `configure()` - - All `AsyncHTTPClient` implementations support a ``defaults`` - keyword argument, which can be used to set default values for - `HTTPRequest` attributes. For example:: - - AsyncHTTPClient.configure( - None, defaults=dict(user_agent="MyUserAgent")) - # or with force_instance: - client = AsyncHTTPClient(force_instance=True, - defaults=dict(user_agent="MyUserAgent")) - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - """ - @classmethod - def configurable_base(cls): - return AsyncHTTPClient - - @classmethod - def configurable_default(cls): - from salt.ext.tornado.simple_httpclient import SimpleAsyncHTTPClient - return SimpleAsyncHTTPClient - - @classmethod - def _async_clients(cls): - attr_name = '_async_client_dict_' + cls.__name__ - if not hasattr(cls, attr_name): - setattr(cls, attr_name, weakref.WeakKeyDictionary()) - return getattr(cls, attr_name) - - def __new__(cls, io_loop=None, force_instance=False, **kwargs): - io_loop = io_loop or IOLoop.current() - if force_instance: - instance_cache = None - else: - instance_cache = cls._async_clients() - if instance_cache is not None and io_loop in instance_cache: - return instance_cache[io_loop] - instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop, - **kwargs) - # Make sure the instance knows which cache to remove itself from. - # It can't simply call _async_clients() because we may be in - # __new__(AsyncHTTPClient) but instance.__class__ may be - # SimpleAsyncHTTPClient. - instance._instance_cache = instance_cache - if instance_cache is not None: - instance_cache[instance.io_loop] = instance - return instance - - def initialize(self, io_loop, defaults=None): - self.io_loop = io_loop - self.defaults = dict(HTTPRequest._DEFAULTS) - if defaults is not None: - self.defaults.update(defaults) - self._closed = False - - def close(self): - """Destroys this HTTP client, freeing any file descriptors used. - - This method is **not needed in normal use** due to the way - that `AsyncHTTPClient` objects are transparently reused. - ``close()`` is generally only necessary when either the - `.IOLoop` is also being closed, or the ``force_instance=True`` - argument was used when creating the `AsyncHTTPClient`. - - No other methods may be called on the `AsyncHTTPClient` after - ``close()``. - - """ - if self._closed: - return - self._closed = True - if self._instance_cache is not None: - if self._instance_cache.get(self.io_loop) is not self: - raise RuntimeError("inconsistent AsyncHTTPClient cache") - del self._instance_cache[self.io_loop] - - def fetch(self, request, callback=None, raise_error=True, **kwargs): - """Executes a request, asynchronously returning an `HTTPResponse`. - - The request may be either a string URL or an `HTTPRequest` object. - If it is a string, we construct an `HTTPRequest` using any additional - kwargs: ``HTTPRequest(request, **kwargs)`` - - This method returns a `.Future` whose result is an - `HTTPResponse`. By default, the ``Future`` will raise an - `HTTPError` if the request returned a non-200 response code - (other errors may also be raised if the server could not be - contacted). Instead, if ``raise_error`` is set to False, the - response will always be returned regardless of the response - code. - - If a ``callback`` is given, it will be invoked with the `HTTPResponse`. - In the callback interface, `HTTPError` is not automatically raised. - Instead, you must check the response's ``error`` attribute or - call its `~HTTPResponse.rethrow` method. - """ - if self._closed: - raise RuntimeError("fetch() called on closed AsyncHTTPClient") - if not isinstance(request, HTTPRequest): - request = HTTPRequest(url=request, **kwargs) - else: - if kwargs: - raise ValueError("kwargs can't be used if request is an HTTPRequest object") - # We may modify this (to add Host, Accept-Encoding, etc), - # so make sure we don't modify the caller's object. This is also - # where normal dicts get converted to HTTPHeaders objects. - request.headers = httputil.HTTPHeaders(request.headers) - request = _RequestProxy(request, self.defaults) - future = TracebackFuture() - if callback is not None: - callback = stack_context.wrap(callback) - - def handle_future(future): - exc = future.exception() - if isinstance(exc, HTTPError) and exc.response is not None: - response = exc.response - elif exc is not None: - response = HTTPResponse( - request, 599, error=exc, - request_time=time.time() - request.start_time) - else: - response = future.result() - self.io_loop.add_callback(callback, response) - future.add_done_callback(handle_future) - - def handle_response(response): - if raise_error and response.error: - future.set_exception(response.error) - else: - future.set_result(response) - self.fetch_impl(request, handle_response) - return future - - def fetch_impl(self, request, callback): - raise NotImplementedError() - - @classmethod - def configure(cls, impl, **kwargs): - """Configures the `AsyncHTTPClient` subclass to use. - - ``AsyncHTTPClient()`` actually creates an instance of a subclass. - This method may be called with either a class object or the - fully-qualified name of such a class (or ``None`` to use the default, - ``SimpleAsyncHTTPClient``) - - If additional keyword arguments are given, they will be passed - to the constructor of each subclass instance created. The - keyword argument ``max_clients`` determines the maximum number - of simultaneous `~AsyncHTTPClient.fetch()` operations that can - execute in parallel on each `.IOLoop`. Additional arguments - may be supported depending on the implementation class in use. - - Example:: - - AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") - """ - super(AsyncHTTPClient, cls).configure(impl, **kwargs) - - -class HTTPRequest(object): - """HTTP client request object.""" - - # Default values for HTTPRequest parameters. - # Merged with the values on the request object by AsyncHTTPClient - # implementations. - _DEFAULTS = dict( - connect_timeout=20.0, - request_timeout=20.0, - follow_redirects=True, - max_redirects=5, - decompress_response=True, - proxy_password='', - allow_nonstandard_methods=False, - validate_cert=True) - - def __init__(self, url, method="GET", headers=None, body=None, - auth_username=None, auth_password=None, auth_mode=None, - connect_timeout=None, request_timeout=None, - if_modified_since=None, follow_redirects=None, - max_redirects=None, user_agent=None, use_gzip=None, - network_interface=None, streaming_callback=None, - header_callback=None, prepare_curl_callback=None, - proxy_host=None, proxy_port=None, proxy_username=None, - proxy_password=None, proxy_auth_mode=None, - allow_nonstandard_methods=None, validate_cert=None, - ca_certs=None, allow_ipv6=None, client_key=None, - client_cert=None, body_producer=None, - expect_100_continue=False, decompress_response=None, - ssl_options=None): - r"""All parameters except ``url`` are optional. - - :arg string url: URL to fetch - :arg string method: HTTP method, e.g. "GET" or "POST" - :arg headers: Additional HTTP headers to pass on the request - :type headers: `~tornado.httputil.HTTPHeaders` or `dict` - :arg body: HTTP request body as a string (byte or unicode; if unicode - the utf-8 encoding will be used) - :arg body_producer: Callable used for lazy/asynchronous request bodies. - It is called with one argument, a ``write`` function, and should - return a `.Future`. It should call the write function with new - data as it becomes available. The write function returns a - `.Future` which can be used for flow control. - Only one of ``body`` and ``body_producer`` may - be specified. ``body_producer`` is not supported on - ``curl_httpclient``. When using ``body_producer`` it is recommended - to pass a ``Content-Length`` in the headers as otherwise chunked - encoding will be used, and many servers do not support chunked - encoding on requests. New in Tornado 4.0 - :arg string auth_username: Username for HTTP authentication - :arg string auth_password: Password for HTTP authentication - :arg string auth_mode: Authentication mode; default is "basic". - Allowed values are implementation-defined; ``curl_httpclient`` - supports "basic" and "digest"; ``simple_httpclient`` only supports - "basic" - :arg float connect_timeout: Timeout for initial connection in seconds, - default 20 seconds - :arg float request_timeout: Timeout for entire request in seconds, - default 20 seconds - :arg if_modified_since: Timestamp for ``If-Modified-Since`` header - :type if_modified_since: `datetime` or `float` - :arg bool follow_redirects: Should redirects be followed automatically - or return the 3xx response? Default True. - :arg int max_redirects: Limit for ``follow_redirects``, default 5. - :arg string user_agent: String to send as ``User-Agent`` header - :arg bool decompress_response: Request a compressed response from - the server and decompress it after downloading. Default is True. - New in Tornado 4.0. - :arg bool use_gzip: Deprecated alias for ``decompress_response`` - since Tornado 4.0. - :arg string network_interface: Network interface to use for request. - ``curl_httpclient`` only; see note below. - :arg callable streaming_callback: If set, ``streaming_callback`` will - be run with each chunk of data as it is received, and - ``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in - the final response. - :arg callable header_callback: If set, ``header_callback`` will - be run with each header line as it is received (including the - first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line - containing only ``\r\n``. All lines include the trailing newline - characters). ``HTTPResponse.headers`` will be empty in the final - response. This is most useful in conjunction with - ``streaming_callback``, because it's the only way to get access to - header data while the request is in progress. - :arg callable prepare_curl_callback: If set, will be called with - a ``pycurl.Curl`` object to allow the application to make additional - ``setopt`` calls. - :arg string proxy_host: HTTP proxy hostname. To use proxies, - ``proxy_host`` and ``proxy_port`` must be set; ``proxy_username``, - ``proxy_pass`` and ``proxy_auth_mode`` are optional. Proxies are - currently only supported with ``curl_httpclient``. - :arg int proxy_port: HTTP proxy port - :arg string proxy_username: HTTP proxy username - :arg string proxy_password: HTTP proxy password - :arg string proxy_auth_mode: HTTP proxy Authentication mode; - default is "basic". supports "basic" and "digest" - :arg bool allow_nonstandard_methods: Allow unknown values for ``method`` - argument? Default is False. - :arg bool validate_cert: For HTTPS requests, validate the server's - certificate? Default is True. - :arg string ca_certs: filename of CA certificates in PEM format, - or None to use defaults. See note below when used with - ``curl_httpclient``. - :arg string client_key: Filename for client SSL key, if any. See - note below when used with ``curl_httpclient``. - :arg string client_cert: Filename for client SSL certificate, if any. - See note below when used with ``curl_httpclient``. - :arg ssl.SSLContext ssl_options: `ssl.SSLContext` object for use in - ``simple_httpclient`` (unsupported by ``curl_httpclient``). - Overrides ``validate_cert``, ``ca_certs``, ``client_key``, - and ``client_cert``. - :arg bool allow_ipv6: Use IPv6 when available? Default is true. - :arg bool expect_100_continue: If true, send the - ``Expect: 100-continue`` header and wait for a continue response - before sending the request body. Only supported with - simple_httpclient. - - .. note:: - - When using ``curl_httpclient`` certain options may be - inherited by subsequent fetches because ``pycurl`` does - not allow them to be cleanly reset. This applies to the - ``ca_certs``, ``client_key``, ``client_cert``, and - ``network_interface`` arguments. If you use these - options, you should pass them on every request (you don't - have to always use the same values, but it's not possible - to mix requests that specify these options with ones that - use the defaults). - - .. versionadded:: 3.1 - The ``auth_mode`` argument. - - .. versionadded:: 4.0 - The ``body_producer`` and ``expect_100_continue`` arguments. - - .. versionadded:: 4.2 - The ``ssl_options`` argument. - - .. versionadded:: 4.5 - The ``proxy_auth_mode`` argument. - """ - # Note that some of these attributes go through property setters - # defined below. - self.headers = headers - if if_modified_since: - self.headers["If-Modified-Since"] = httputil.format_timestamp( - if_modified_since) - self.proxy_host = proxy_host - self.proxy_port = proxy_port - self.proxy_username = proxy_username - self.proxy_password = proxy_password - self.proxy_auth_mode = proxy_auth_mode - self.url = url - self.method = method - self.body = body - self.body_producer = body_producer - self.auth_username = auth_username - self.auth_password = auth_password - self.auth_mode = auth_mode - self.connect_timeout = connect_timeout - self.request_timeout = request_timeout - self.follow_redirects = follow_redirects - self.max_redirects = max_redirects - self.user_agent = user_agent - if decompress_response is not None: - self.decompress_response = decompress_response - else: - self.decompress_response = use_gzip - self.network_interface = network_interface - self.streaming_callback = streaming_callback - self.header_callback = header_callback - self.prepare_curl_callback = prepare_curl_callback - self.allow_nonstandard_methods = allow_nonstandard_methods - self.validate_cert = validate_cert - self.ca_certs = ca_certs - self.allow_ipv6 = allow_ipv6 - self.client_key = client_key - self.client_cert = client_cert - self.ssl_options = ssl_options - self.expect_100_continue = expect_100_continue - self.start_time = time.time() - - @property - def headers(self): - return self._headers - - @headers.setter - def headers(self, value): - if value is None: - self._headers = httputil.HTTPHeaders() - else: - self._headers = value - - @property - def body(self): - return self._body - - @body.setter - def body(self, value): - self._body = utf8(value) - - @property - def body_producer(self): - return self._body_producer - - @body_producer.setter - def body_producer(self, value): - self._body_producer = stack_context.wrap(value) - - @property - def streaming_callback(self): - return self._streaming_callback - - @streaming_callback.setter - def streaming_callback(self, value): - self._streaming_callback = stack_context.wrap(value) - - @property - def header_callback(self): - return self._header_callback - - @header_callback.setter - def header_callback(self, value): - self._header_callback = stack_context.wrap(value) - - @property - def prepare_curl_callback(self): - return self._prepare_curl_callback - - @prepare_curl_callback.setter - def prepare_curl_callback(self, value): - self._prepare_curl_callback = stack_context.wrap(value) - - -class HTTPResponse(object): - """HTTP Response object. - - Attributes: - - * request: HTTPRequest object - - * code: numeric HTTP status code, e.g. 200 or 404 - - * reason: human-readable reason phrase describing the status code - - * headers: `tornado.httputil.HTTPHeaders` object - - * effective_url: final location of the resource after following any - redirects - - * buffer: ``cStringIO`` object for response body - - * body: response body as bytes (created on demand from ``self.buffer``) - - * error: Exception object, if any - - * request_time: seconds from request start to finish - - * time_info: dictionary of diagnostic timing information from the request. - Available data are subject to change, but currently uses timings - available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html, - plus ``queue``, which is the delay (if any) introduced by waiting for - a slot under `AsyncHTTPClient`'s ``max_clients`` setting. - """ - def __init__(self, request, code, headers=None, buffer=None, - effective_url=None, error=None, request_time=None, - time_info=None, reason=None): - if isinstance(request, _RequestProxy): - self.request = request.request - else: - self.request = request - self.code = code - self.reason = reason or httputil.responses.get(code, "Unknown") - if headers is not None: - self.headers = headers - else: - self.headers = httputil.HTTPHeaders() - self.buffer = buffer - self._body = None - if effective_url is None: - self.effective_url = request.url - else: - self.effective_url = effective_url - if error is None: - if self.code < 200 or self.code >= 300: - self.error = HTTPError(self.code, message=self.reason, - response=self) - else: - self.error = None - else: - self.error = error - self.request_time = request_time - self.time_info = time_info or {} - - @property - def body(self): - if self.buffer is None: - return None - elif self._body is None: - self._body = self.buffer.getvalue() - - return self._body - - def rethrow(self): - """If there was an error on the request, raise an `HTTPError`.""" - if self.error: - raise self.error - - def __repr__(self): - args = ",".join("%s=%r" % i for i in sorted(self.__dict__.items())) - return "%s(%s)" % (self.__class__.__name__, args) - - -class HTTPError(Exception): - """Exception thrown for an unsuccessful HTTP request. - - Attributes: - - * ``code`` - HTTP error integer error code, e.g. 404. Error code 599 is - used when no HTTP response was received, e.g. for a timeout. - - * ``response`` - `HTTPResponse` object, if any. - - Note that if ``follow_redirects`` is False, redirects become HTTPErrors, - and you can look at ``error.response.headers['Location']`` to see the - destination of the redirect. - """ - def __init__(self, code, message=None, response=None): - self.code = code - self.message = message or httputil.responses.get(code, "Unknown") - self.response = response - super(HTTPError, self).__init__(code, message, response) - - def __str__(self): - return "HTTP %d: %s" % (self.code, self.message) - - # There is a cyclic reference between self and self.response, - # which breaks the default __repr__ implementation. - # (especially on pypy, which doesn't have the same recursion - # detection as cpython). - __repr__ = __str__ - - -class _RequestProxy(object): - """Combines an object with a dictionary of defaults. - - Used internally by AsyncHTTPClient implementations. - """ - def __init__(self, request, defaults): - self.request = request - self.defaults = defaults - - def __getattr__(self, name): - request_attr = getattr(self.request, name) - if request_attr is not None: - return request_attr - elif self.defaults is not None: - return self.defaults.get(name, None) - else: - return None - - -def main(): - from salt.ext.tornado.options import define, options, parse_command_line - define("print_headers", type=bool, default=False) - define("print_body", type=bool, default=True) - define("follow_redirects", type=bool, default=True) - define("validate_cert", type=bool, default=True) - args = parse_command_line() - client = HTTPClient() - for arg in args: - try: - response = client.fetch(arg, - follow_redirects=options.follow_redirects, - validate_cert=options.validate_cert, - ) - except HTTPError as e: - if e.response is not None: - response = e.response - else: - raise - if options.print_headers: - print(response.headers) - if options.print_body: - print(native_str(response.body)) - client.close() - - -if __name__ == "__main__": - main() diff --git a/salt/ext/tornado/httpserver.py b/salt/ext/tornado/httpserver.py deleted file mode 100644 index 06a5ffb46e5..00000000000 --- a/salt/ext/tornado/httpserver.py +++ /dev/null @@ -1,326 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A non-blocking, single-threaded HTTP server. - -Typical applications have little direct interaction with the `HTTPServer` -class except to start a server at the beginning of the process -(and even that is often done indirectly via `tornado.web.Application.listen`). - -.. versionchanged:: 4.0 - - The ``HTTPRequest`` class that used to live in this module has been moved - to `tornado.httputil.HTTPServerRequest`. The old name remains as an alias. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import socket - -from salt.ext.tornado.escape import native_str -from salt.ext.tornado.http1connection import HTTP1ServerConnection, HTTP1ConnectionParameters -from salt.ext.tornado import gen -from salt.ext.tornado import httputil -from salt.ext.tornado import iostream -from salt.ext.tornado import netutil -from salt.ext.tornado.tcpserver import TCPServer -from salt.ext.tornado.util import Configurable - - -class HTTPServer(TCPServer, Configurable, - httputil.HTTPServerConnectionDelegate): - r"""A non-blocking, single-threaded HTTP server. - - A server is defined by a subclass of `.HTTPServerConnectionDelegate`, - or, for backwards compatibility, a callback that takes an - `.HTTPServerRequest` as an argument. The delegate is usually a - `tornado.web.Application`. - - `HTTPServer` supports keep-alive connections by default - (automatically for HTTP/1.1, or for HTTP/1.0 when the client - requests ``Connection: keep-alive``). - - If ``xheaders`` is ``True``, we support the - ``X-Real-Ip``/``X-Forwarded-For`` and - ``X-Scheme``/``X-Forwarded-Proto`` headers, which override the - remote IP and URI scheme/protocol for all requests. These headers - are useful when running Tornado behind a reverse proxy or load - balancer. The ``protocol`` argument can also be set to ``https`` - if Tornado is run behind an SSL-decoding proxy that does not set one of - the supported ``xheaders``. - - By default, when parsing the ``X-Forwarded-For`` header, Tornado will - select the last (i.e., the closest) address on the list of hosts as the - remote host IP address. To select the next server in the chain, a list of - trusted downstream hosts may be passed as the ``trusted_downstream`` - argument. These hosts will be skipped when parsing the ``X-Forwarded-For`` - header. - - To make this server serve SSL traffic, send the ``ssl_options`` keyword - argument with an `ssl.SSLContext` object. For compatibility with older - versions of Python ``ssl_options`` may also be a dictionary of keyword - arguments for the `ssl.wrap_socket` method.:: - - ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ssl_ctx.load_cert_chain(os.path.join(data_dir, "mydomain.crt"), - os.path.join(data_dir, "mydomain.key")) - HTTPServer(applicaton, ssl_options=ssl_ctx) - - `HTTPServer` initialization follows one of three patterns (the - initialization methods are defined on `tornado.tcpserver.TCPServer`): - - 1. `~tornado.tcpserver.TCPServer.listen`: simple single-process:: - - server = HTTPServer(app) - server.listen(8888) - IOLoop.current().start() - - In many cases, `tornado.web.Application.listen` can be used to avoid - the need to explicitly create the `HTTPServer`. - - 2. `~tornado.tcpserver.TCPServer.bind`/`~tornado.tcpserver.TCPServer.start`: - simple multi-process:: - - server = HTTPServer(app) - server.bind(8888) - server.start(0) # Forks multiple sub-processes - IOLoop.current().start() - - When using this interface, an `.IOLoop` must *not* be passed - to the `HTTPServer` constructor. `~.TCPServer.start` will always start - the server on the default singleton `.IOLoop`. - - 3. `~tornado.tcpserver.TCPServer.add_sockets`: advanced multi-process:: - - sockets = tornado.netutil.bind_sockets(8888) - tornado.process.fork_processes(0) - server = HTTPServer(app) - server.add_sockets(sockets) - IOLoop.current().start() - - The `~.TCPServer.add_sockets` interface is more complicated, - but it can be used with `tornado.process.fork_processes` to - give you more flexibility in when the fork happens. - `~.TCPServer.add_sockets` can also be used in single-process - servers if you want to create your listening sockets in some - way other than `tornado.netutil.bind_sockets`. - - .. versionchanged:: 4.0 - Added ``decompress_request``, ``chunk_size``, ``max_header_size``, - ``idle_connection_timeout``, ``body_timeout``, ``max_body_size`` - arguments. Added support for `.HTTPServerConnectionDelegate` - instances as ``request_callback``. - - .. versionchanged:: 4.1 - `.HTTPServerConnectionDelegate.start_request` is now called with - two arguments ``(server_conn, request_conn)`` (in accordance with the - documentation) instead of one ``(request_conn)``. - - .. versionchanged:: 4.2 - `HTTPServer` is now a subclass of `tornado.util.Configurable`. - - .. versionchanged:: 4.5 - Added the ``trusted_downstream`` argument. - """ - def __init__(self, *args, **kwargs): - # Ignore args to __init__; real initialization belongs in - # initialize since we're Configurable. (there's something - # weird in initialization order between this class, - # Configurable, and TCPServer so we can't leave __init__ out - # completely) - pass - - def initialize(self, request_callback, no_keep_alive=False, io_loop=None, - xheaders=False, ssl_options=None, protocol=None, - decompress_request=False, - chunk_size=None, max_header_size=None, - idle_connection_timeout=None, body_timeout=None, - max_body_size=None, max_buffer_size=None, - trusted_downstream=None): - self.request_callback = request_callback - self.no_keep_alive = no_keep_alive - self.xheaders = xheaders - self.protocol = protocol - self.conn_params = HTTP1ConnectionParameters( - decompress=decompress_request, - chunk_size=chunk_size, - max_header_size=max_header_size, - header_timeout=idle_connection_timeout or 3600, - max_body_size=max_body_size, - body_timeout=body_timeout, - no_keep_alive=no_keep_alive) - TCPServer.__init__(self, io_loop=io_loop, ssl_options=ssl_options, - max_buffer_size=max_buffer_size, - read_chunk_size=chunk_size) - self._connections = set() - self.trusted_downstream = trusted_downstream - - @classmethod - def configurable_base(cls): - return HTTPServer - - @classmethod - def configurable_default(cls): - return HTTPServer - - @gen.coroutine - def close_all_connections(self): - while self._connections: - # Peek at an arbitrary element of the set - conn = next(iter(self._connections)) - yield conn.close() - - def handle_stream(self, stream, address): - context = _HTTPRequestContext(stream, address, - self.protocol, - self.trusted_downstream) - conn = HTTP1ServerConnection( - stream, self.conn_params, context) - self._connections.add(conn) - conn.start_serving(self) - - def start_request(self, server_conn, request_conn): - if isinstance(self.request_callback, httputil.HTTPServerConnectionDelegate): - delegate = self.request_callback.start_request(server_conn, request_conn) - else: - delegate = _CallableAdapter(self.request_callback, request_conn) - - if self.xheaders: - delegate = _ProxyAdapter(delegate, request_conn) - - return delegate - - def on_close(self, server_conn): - self._connections.remove(server_conn) - - -class _CallableAdapter(httputil.HTTPMessageDelegate): - def __init__(self, request_callback, request_conn): - self.connection = request_conn - self.request_callback = request_callback - self.request = None - self.delegate = None - self._chunks = [] - - def headers_received(self, start_line, headers): - self.request = httputil.HTTPServerRequest( - connection=self.connection, start_line=start_line, - headers=headers) - - def data_received(self, chunk): - self._chunks.append(chunk) - - def finish(self): - self.request.body = b''.join(self._chunks) - self.request._parse_body() - self.request_callback(self.request) - - def on_connection_close(self): - self._chunks = None - - -class _HTTPRequestContext(object): - def __init__(self, stream, address, protocol, trusted_downstream=None): - self.address = address - # Save the socket's address family now so we know how to - # interpret self.address even after the stream is closed - # and its socket attribute replaced with None. - if stream.socket is not None: - self.address_family = stream.socket.family - else: - self.address_family = None - # In HTTPServerRequest we want an IP, not a full socket address. - if (self.address_family in (socket.AF_INET, socket.AF_INET6) and - address is not None): - self.remote_ip = address[0] - else: - # Unix (or other) socket; fake the remote address. - self.remote_ip = '0.0.0.0' - if protocol: - self.protocol = protocol - elif isinstance(stream, iostream.SSLIOStream): - self.protocol = "https" - else: - self.protocol = "http" - self._orig_remote_ip = self.remote_ip - self._orig_protocol = self.protocol - self.trusted_downstream = set(trusted_downstream or []) - - def __str__(self): - if self.address_family in (socket.AF_INET, socket.AF_INET6): - return self.remote_ip - elif isinstance(self.address, bytes): - # Python 3 with the -bb option warns about str(bytes), - # so convert it explicitly. - # Unix socket addresses are str on mac but bytes on linux. - return native_str(self.address) - else: - return str(self.address) - - def _apply_xheaders(self, headers): - """Rewrite the ``remote_ip`` and ``protocol`` fields.""" - # Squid uses X-Forwarded-For, others use X-Real-Ip - ip = headers.get("X-Forwarded-For", self.remote_ip) - # Skip trusted downstream hosts in X-Forwarded-For list - for ip in (cand.strip() for cand in reversed(ip.split(','))): - if ip not in self.trusted_downstream: - break - ip = headers.get("X-Real-Ip", ip) - if netutil.is_valid_ip(ip): - self.remote_ip = ip - # AWS uses X-Forwarded-Proto - proto_header = headers.get( - "X-Scheme", headers.get("X-Forwarded-Proto", - self.protocol)) - if proto_header in ("http", "https"): - self.protocol = proto_header - - def _unapply_xheaders(self): - """Undo changes from `_apply_xheaders`. - - Xheaders are per-request so they should not leak to the next - request on the same connection. - """ - self.remote_ip = self._orig_remote_ip - self.protocol = self._orig_protocol - - -class _ProxyAdapter(httputil.HTTPMessageDelegate): - def __init__(self, delegate, request_conn): - self.connection = request_conn - self.delegate = delegate - - def headers_received(self, start_line, headers): - self.connection.context._apply_xheaders(headers) - return self.delegate.headers_received(start_line, headers) - - def data_received(self, chunk): - return self.delegate.data_received(chunk) - - def finish(self): - self.delegate.finish() - self._cleanup() - - def on_connection_close(self): - self.delegate.on_connection_close() - self._cleanup() - - def _cleanup(self): - self.connection.context._unapply_xheaders() - - -HTTPRequest = httputil.HTTPServerRequest diff --git a/salt/ext/tornado/httputil.py b/salt/ext/tornado/httputil.py deleted file mode 100644 index c7a5ac7c3c8..00000000000 --- a/salt/ext/tornado/httputil.py +++ /dev/null @@ -1,1058 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""HTTP utility code shared by clients and servers. - -This module also defines the `HTTPServerRequest` class which is exposed -via `tornado.web.RequestHandler.request`. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import calendar -import collections -import copy -import datetime -import email.utils -import numbers -import re -import time -from collections.abc import MutableMapping - -from salt.ext.tornado.escape import native_str, parse_qs_bytes, utf8 -from salt.ext.tornado.log import gen_log -from salt.ext.tornado.util import PY3, ObjectDict - -if PY3: - import http.cookies as Cookie - from http.client import responses - from urllib.parse import urlencode, urlparse, urlunparse, parse_qsl -else: - import Cookie - from httplib import responses - from urllib import urlencode - from urlparse import urlparse, urlunparse, parse_qsl - - -# responses is unused in this file, but we re-export it to other files. -# Reference it so pyflakes doesn't complain. -responses - -try: - from ssl import SSLError -except ImportError: - # ssl is unavailable on app engine. - class _SSLError(Exception): - pass - - # Hack around a mypy limitation. We can't simply put "type: ignore" - # on the class definition itself; must go through an assignment. - SSLError = _SSLError # type: ignore - -try: - import typing -except ImportError: - pass - - -# RFC 7230 section 3.5: a recipient MAY recognize a single LF as a line -# terminator and ignore any preceding CR. -_CRLF_RE = re.compile(r"\r?\n") - - -class _NormalizedHeaderCache(dict): - """Dynamic cached mapping of header names to Http-Header-Case. - - Implemented as a dict subclass so that cache hits are as fast as a - normal dict lookup, without the overhead of a python function - call. - - >>> normalized_headers = _NormalizedHeaderCache(10) - >>> normalized_headers["coNtent-TYPE"] - 'Content-Type' - """ - - def __init__(self, size): - super(_NormalizedHeaderCache, self).__init__() - self.size = size - self.queue = collections.deque() - - def __missing__(self, key): - normalized = "-".join([w.capitalize() for w in key.split("-")]) - self[key] = normalized - self.queue.append(key) - if len(self.queue) > self.size: - # Limit the size of the cache. LRU would be better, but this - # simpler approach should be fine. In Python 2.7+ we could - # use OrderedDict (or in 3.2+, @functools.lru_cache). - old_key = self.queue.popleft() - del self[old_key] - return normalized - - -_normalized_headers = _NormalizedHeaderCache(1000) - - -class HTTPHeaders(MutableMapping): - """A dictionary that maintains ``Http-Header-Case`` for all keys. - - Supports multiple values per key via a pair of new methods, - `add()` and `get_list()`. The regular dictionary interface - returns a single value per key, with multiple values joined by a - comma. - - >>> h = HTTPHeaders({"content-type": "text/html"}) - >>> list(h.keys()) - ['Content-Type'] - >>> h["Content-Type"] - 'text/html' - - >>> h.add("Set-Cookie", "A=B") - >>> h.add("Set-Cookie", "C=D") - >>> h["set-cookie"] - 'A=B,C=D' - >>> h.get_list("set-cookie") - ['A=B', 'C=D'] - - >>> for (k,v) in sorted(h.get_all()): - ... print('%s: %s' % (k,v)) - ... - Content-Type: text/html - Set-Cookie: A=B - Set-Cookie: C=D - """ - - def __init__(self, *args, **kwargs): - self._dict = {} # type: typing.Dict[str, str] - self._as_list = {} # type: typing.Dict[str, typing.List[str]] - self._last_key = None - if len(args) == 1 and len(kwargs) == 0 and isinstance(args[0], HTTPHeaders): - # Copy constructor - for k, v in args[0].get_all(): - self.add(k, v) - else: - # Dict-style initialization - self.update(*args, **kwargs) - - # new public methods - - def add(self, name, value): - # type: (str, str) -> None - """Adds a new value for the given key.""" - norm_name = _normalized_headers[name] - self._last_key = norm_name - if norm_name in self: - self._dict[norm_name] = ( - native_str(self[norm_name]) + "," + native_str(value) - ) - self._as_list[norm_name].append(value) - else: - self[norm_name] = value - - def get_list(self, name): - """Returns all values for the given header as a list.""" - norm_name = _normalized_headers[name] - return self._as_list.get(norm_name, []) - - def get_all(self): - # type: () -> typing.Iterable[typing.Tuple[str, str]] - """Returns an iterable of all (name, value) pairs. - - If a header has multiple values, multiple pairs will be - returned with the same name. - """ - for name, values in self._as_list.items(): - for value in values: - yield (name, value) - - def parse_line(self, line): - """Updates the dictionary with a single header line. - - >>> h = HTTPHeaders() - >>> h.parse_line("Content-Type: text/html") - >>> h.get('content-type') - 'text/html' - """ - if line[0].isspace(): - # continuation of a multi-line header - new_part = " " + line.lstrip() - self._as_list[self._last_key][-1] += new_part - self._dict[self._last_key] += new_part - else: - name, value = line.split(":", 1) - self.add(name, value.strip()) - - @classmethod - def parse(cls, headers): - """Returns a dictionary from HTTP header text. - - >>> h = HTTPHeaders.parse("Content-Type: text/html\\r\\nContent-Length: 42\\r\\n") - >>> sorted(h.items()) - [('Content-Length', '42'), ('Content-Type', 'text/html')] - """ - h = cls() - for line in _CRLF_RE.split(headers): - if line: - h.parse_line(line) - return h - - # MutableMapping abstract method implementations. - - def __setitem__(self, name, value): - norm_name = _normalized_headers[name] - self._dict[norm_name] = value - self._as_list[norm_name] = [value] - - def __getitem__(self, name): - # type: (str) -> str - return self._dict[_normalized_headers[name]] - - def __delitem__(self, name): - norm_name = _normalized_headers[name] - del self._dict[norm_name] - del self._as_list[norm_name] - - def __len__(self): - return len(self._dict) - - def __iter__(self): - return iter(self._dict) - - def copy(self): - # defined in dict but not in MutableMapping. - return HTTPHeaders(self) - - # Use our overridden copy method for the copy.copy module. - # This makes shallow copies one level deeper, but preserves - # the appearance that HTTPHeaders is a single container. - __copy__ = copy - - def __str__(self): - lines = [] - for name, value in self.get_all(): - lines.append("%s: %s\n" % (name, value)) - return "".join(lines) - - __unicode__ = __str__ - - -class HTTPServerRequest(object): - """A single HTTP request. - - All attributes are type `str` unless otherwise noted. - - .. attribute:: method - - HTTP request method, e.g. "GET" or "POST" - - .. attribute:: uri - - The requested uri. - - .. attribute:: path - - The path portion of `uri` - - .. attribute:: query - - The query portion of `uri` - - .. attribute:: version - - HTTP version specified in request, e.g. "HTTP/1.1" - - .. attribute:: headers - - `.HTTPHeaders` dictionary-like object for request headers. Acts like - a case-insensitive dictionary with additional methods for repeated - headers. - - .. attribute:: body - - Request body, if present, as a byte string. - - .. attribute:: remote_ip - - Client's IP address as a string. If ``HTTPServer.xheaders`` is set, - will pass along the real IP address provided by a load balancer - in the ``X-Real-Ip`` or ``X-Forwarded-For`` header. - - .. versionchanged:: 3.1 - The list format of ``X-Forwarded-For`` is now supported. - - .. attribute:: protocol - - The protocol used, either "http" or "https". If ``HTTPServer.xheaders`` - is set, will pass along the protocol used by a load balancer if - reported via an ``X-Scheme`` header. - - .. attribute:: host - - The requested hostname, usually taken from the ``Host`` header. - - .. attribute:: arguments - - GET/POST arguments are available in the arguments property, which - maps arguments names to lists of values (to support multiple values - for individual names). Names are of type `str`, while arguments - are byte strings. Note that this is different from - `.RequestHandler.get_argument`, which returns argument values as - unicode strings. - - .. attribute:: query_arguments - - Same format as ``arguments``, but contains only arguments extracted - from the query string. - - .. versionadded:: 3.2 - - .. attribute:: body_arguments - - Same format as ``arguments``, but contains only arguments extracted - from the request body. - - .. versionadded:: 3.2 - - .. attribute:: files - - File uploads are available in the files property, which maps file - names to lists of `.HTTPFile`. - - .. attribute:: connection - - An HTTP request is attached to a single HTTP connection, which can - be accessed through the "connection" attribute. Since connections - are typically kept open in HTTP/1.1, multiple requests can be handled - sequentially on a single connection. - - .. versionchanged:: 4.0 - Moved from ``tornado.httpserver.HTTPRequest``. - """ - - def __init__( - self, - method=None, - uri=None, - version="HTTP/1.0", - headers=None, - body=None, - host=None, - files=None, - connection=None, - start_line=None, - server_connection=None, - ): - if start_line is not None: - method, uri, version = start_line - self.method = method - self.uri = uri - self.version = version - self.headers = headers or HTTPHeaders() - self.body = body or b"" - - # set remote IP and protocol - context = getattr(connection, "context", None) - self.remote_ip = getattr(context, "remote_ip", None) - self.protocol = getattr(context, "protocol", "http") - - self.host = host or self.headers.get("Host") or "127.0.0.1" - self.host_name = split_host_and_port(self.host.lower())[0] - self.files = files or {} - self.connection = connection - self.server_connection = server_connection - self._start_time = time.time() - self._finish_time = None - - self.path, sep, self.query = uri.partition("?") - self.arguments = parse_qs_bytes(self.query, keep_blank_values=True) - self.query_arguments = copy.deepcopy(self.arguments) - self.body_arguments = {} - - def supports_http_1_1(self): - """Returns True if this request supports HTTP/1.1 semantics. - - .. deprecated:: 4.0 - Applications are less likely to need this information with the - introduction of `.HTTPConnection`. If you still need it, access - the ``version`` attribute directly. - """ - return self.version == "HTTP/1.1" - - @property - def cookies(self): - """A dictionary of Cookie.Morsel objects.""" - if not hasattr(self, "_cookies"): - self._cookies = Cookie.SimpleCookie() - if "Cookie" in self.headers: - try: - parsed = parse_cookie(self.headers["Cookie"]) - except Exception: - pass - else: - for k, v in parsed.items(): - try: - self._cookies[k] = v - except Exception: - # SimpleCookie imposes some restrictions on keys; - # parse_cookie does not. Discard any cookies - # with disallowed keys. - pass - return self._cookies - - def write(self, chunk, callback=None): - """Writes the given chunk to the response stream. - - .. deprecated:: 4.0 - Use ``request.connection`` and the `.HTTPConnection` methods - to write the response. - """ - assert isinstance(chunk, bytes) - assert self.version.startswith( - "HTTP/1." - ), "deprecated interface only supported in HTTP/1.x" - self.connection.write(chunk, callback=callback) - - def finish(self): - """Finishes this HTTP request on the open connection. - - .. deprecated:: 4.0 - Use ``request.connection`` and the `.HTTPConnection` methods - to write the response. - """ - self.connection.finish() - self._finish_time = time.time() - - def full_url(self): - """Reconstructs the full URL for this request.""" - return self.protocol + "://" + self.host + self.uri - - def request_time(self): - """Returns the amount of time it took for this request to execute.""" - if self._finish_time is None: - return time.time() - self._start_time - else: - return self._finish_time - self._start_time - - def get_ssl_certificate(self, binary_form=False): - """Returns the client's SSL certificate, if any. - - To use client certificates, the HTTPServer's - `ssl.SSLContext.verify_mode` field must be set, e.g.:: - - ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ssl_ctx.load_cert_chain("foo.crt", "foo.key") - ssl_ctx.load_verify_locations("cacerts.pem") - ssl_ctx.verify_mode = ssl.CERT_REQUIRED - server = HTTPServer(app, ssl_options=ssl_ctx) - - By default, the return value is a dictionary (or None, if no - client certificate is present). If ``binary_form`` is true, a - DER-encoded form of the certificate is returned instead. See - SSLSocket.getpeercert() in the standard library for more - details. - http://docs.python.org/library/ssl.html#sslsocket-objects - """ - try: - return self.connection.stream.socket.getpeercert(binary_form=binary_form) - except SSLError: - return None - - def _parse_body(self): - parse_body_arguments( - self.headers.get("Content-Type", ""), - self.body, - self.body_arguments, - self.files, - self.headers, - ) - - for k, v in self.body_arguments.items(): - self.arguments.setdefault(k, []).extend(v) - - def __repr__(self): - attrs = ("protocol", "host", "method", "uri", "version", "remote_ip") - args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs]) - return "%s(%s, headers=%s)" % ( - self.__class__.__name__, - args, - dict(self.headers), - ) - - -class HTTPInputError(Exception): - """Exception class for malformed HTTP requests or responses - from remote sources. - - .. versionadded:: 4.0 - """ - - pass - - -class HTTPOutputError(Exception): - """Exception class for errors in HTTP output. - - .. versionadded:: 4.0 - """ - - pass - - -class HTTPServerConnectionDelegate(object): - """Implement this interface to handle requests from `.HTTPServer`. - - .. versionadded:: 4.0 - """ - - def start_request(self, server_conn, request_conn): - """This method is called by the server when a new request has started. - - :arg server_conn: is an opaque object representing the long-lived - (e.g. tcp-level) connection. - :arg request_conn: is a `.HTTPConnection` object for a single - request/response exchange. - - This method should return a `.HTTPMessageDelegate`. - """ - raise NotImplementedError() - - def on_close(self, server_conn): - """This method is called when a connection has been closed. - - :arg server_conn: is a server connection that has previously been - passed to ``start_request``. - """ - pass - - -class HTTPMessageDelegate(object): - """Implement this interface to handle an HTTP request or response. - - .. versionadded:: 4.0 - """ - - def headers_received(self, start_line, headers): - """Called when the HTTP headers have been received and parsed. - - :arg start_line: a `.RequestStartLine` or `.ResponseStartLine` - depending on whether this is a client or server message. - :arg headers: a `.HTTPHeaders` instance. - - Some `.HTTPConnection` methods can only be called during - ``headers_received``. - - May return a `.Future`; if it does the body will not be read - until it is done. - """ - pass - - def data_received(self, chunk): - """Called when a chunk of data has been received. - - May return a `.Future` for flow control. - """ - pass - - def finish(self): - """Called after the last chunk of data has been received.""" - pass - - def on_connection_close(self): - """Called if the connection is closed without finishing the request. - - If ``headers_received`` is called, either ``finish`` or - ``on_connection_close`` will be called, but not both. - """ - pass - - -class HTTPConnection(object): - """Applications use this interface to write their responses. - - .. versionadded:: 4.0 - """ - - def write_headers(self, start_line, headers, chunk=None, callback=None): - """Write an HTTP header block. - - :arg start_line: a `.RequestStartLine` or `.ResponseStartLine`. - :arg headers: a `.HTTPHeaders` instance. - :arg chunk: the first (optional) chunk of data. This is an optimization - so that small responses can be written in the same call as their - headers. - :arg callback: a callback to be run when the write is complete. - - The ``version`` field of ``start_line`` is ignored. - - Returns a `.Future` if no callback is given. - """ - raise NotImplementedError() - - def write(self, chunk, callback=None): - """Writes a chunk of body data. - - The callback will be run when the write is complete. If no callback - is given, returns a Future. - """ - raise NotImplementedError() - - def finish(self): - """Indicates that the last body data has been written. - """ - raise NotImplementedError() - - -def url_concat(url, args): - """Concatenate url and arguments regardless of whether - url has existing query parameters. - - ``args`` may be either a dictionary or a list of key-value pairs - (the latter allows for multiple values with the same key. - - >>> url_concat("http://example.com/foo", dict(c="d")) - 'http://example.com/foo?c=d' - >>> url_concat("http://example.com/foo?a=b", dict(c="d")) - 'http://example.com/foo?a=b&c=d' - >>> url_concat("http://example.com/foo?a=b", [("c", "d"), ("c", "d2")]) - 'http://example.com/foo?a=b&c=d&c=d2' - """ - if args is None: - return url - parsed_url = urlparse(url) - if isinstance(args, dict): - parsed_query = parse_qsl(parsed_url.query, keep_blank_values=True) - parsed_query.extend(args.items()) - elif isinstance(args, list) or isinstance(args, tuple): - parsed_query = parse_qsl(parsed_url.query, keep_blank_values=True) - parsed_query.extend(args) - else: - err = "'args' parameter should be dict, list or tuple. Not {0}".format( - type(args) - ) - raise TypeError(err) - final_query = urlencode(parsed_query) - url = urlunparse( - ( - parsed_url[0], - parsed_url[1], - parsed_url[2], - parsed_url[3], - final_query, - parsed_url[5], - ) - ) - return url - - -class HTTPFile(ObjectDict): - """Represents a file uploaded via a form. - - For backwards compatibility, its instance attributes are also - accessible as dictionary keys. - - * ``filename`` - * ``body`` - * ``content_type`` - """ - - pass - - -def _parse_request_range(range_header): - """Parses a Range header. - - Returns either ``None`` or tuple ``(start, end)``. - Note that while the HTTP headers use inclusive byte positions, - this method returns indexes suitable for use in slices. - - >>> start, end = _parse_request_range("bytes=1-2") - >>> start, end - (1, 3) - >>> [0, 1, 2, 3, 4][start:end] - [1, 2] - >>> _parse_request_range("bytes=6-") - (6, None) - >>> _parse_request_range("bytes=-6") - (-6, None) - >>> _parse_request_range("bytes=-0") - (None, 0) - >>> _parse_request_range("bytes=") - (None, None) - >>> _parse_request_range("foo=42") - >>> _parse_request_range("bytes=1-2,6-10") - - Note: only supports one range (ex, ``bytes=1-2,6-10`` is not allowed). - - See [0] for the details of the range header. - - [0]: http://greenbytes.de/tech/webdav/draft-ietf-httpbis-p5-range-latest.html#byte.ranges - """ - unit, _, value = range_header.partition("=") - unit, value = unit.strip(), value.strip() - if unit != "bytes": - return None - start_b, _, end_b = value.partition("-") - try: - start = _int_or_none(start_b) - end = _int_or_none(end_b) - except ValueError: - return None - if end is not None: - if start is None: - if end != 0: - start = -end - end = None - else: - end += 1 - return (start, end) - - -def _get_content_range(start, end, total): - """Returns a suitable Content-Range header: - - >>> print(_get_content_range(None, 1, 4)) - bytes 0-0/4 - >>> print(_get_content_range(1, 3, 4)) - bytes 1-2/4 - >>> print(_get_content_range(None, None, 4)) - bytes 0-3/4 - """ - start = start or 0 - end = (end or total) - 1 - return "bytes %s-%s/%s" % (start, end, total) - - -def _int_or_none(val): - val = val.strip() - if val == "": - return None - return int(val) - - -def parse_body_arguments(content_type, body, arguments, files, headers=None): - """Parses a form request body. - - Supports ``application/x-www-form-urlencoded`` and - ``multipart/form-data``. The ``content_type`` parameter should be - a string and ``body`` should be a byte string. The ``arguments`` - and ``files`` parameters are dictionaries that will be updated - with the parsed contents. - """ - if headers and "Content-Encoding" in headers: - gen_log.warning("Unsupported Content-Encoding: %s", headers["Content-Encoding"]) - return - if content_type.startswith("application/x-www-form-urlencoded"): - try: - uri_arguments = parse_qs_bytes(native_str(body), keep_blank_values=True) - except Exception as e: - gen_log.warning("Invalid x-www-form-urlencoded body: %s", e) - uri_arguments = {} - for name, values in uri_arguments.items(): - if values: - arguments.setdefault(name, []).extend(values) - elif content_type.startswith("multipart/form-data"): - try: - fields = content_type.split(";") - for field in fields: - k, sep, v = field.strip().partition("=") - if k == "boundary" and v: - parse_multipart_form_data(utf8(v), body, arguments, files) - break - else: - raise ValueError("multipart boundary not found") - except Exception as e: - gen_log.warning("Invalid multipart/form-data: %s", e) - - -def parse_multipart_form_data(boundary, data, arguments, files): - """Parses a ``multipart/form-data`` body. - - The ``boundary`` and ``data`` parameters are both byte strings. - The dictionaries given in the arguments and files parameters - will be updated with the contents of the body. - """ - # The standard allows for the boundary to be quoted in the header, - # although it's rare (it happens at least for google app engine - # xmpp). I think we're also supposed to handle backslash-escapes - # here but I'll save that until we see a client that uses them - # in the wild. - if boundary.startswith(b'"') and boundary.endswith(b'"'): - boundary = boundary[1:-1] - final_boundary_index = data.rfind(b"--" + boundary + b"--") - if final_boundary_index == -1: - gen_log.warning("Invalid multipart/form-data: no final boundary") - return - parts = data[:final_boundary_index].split(b"--" + boundary + b"\r\n") - for part in parts: - if not part: - continue - eoh = part.find(b"\r\n\r\n") - if eoh == -1: - gen_log.warning("multipart/form-data missing headers") - continue - headers = HTTPHeaders.parse(part[:eoh].decode("utf-8")) - disp_header = headers.get("Content-Disposition", "") - disposition, disp_params = _parse_header(disp_header) - if disposition != "form-data" or not part.endswith(b"\r\n"): - gen_log.warning("Invalid multipart/form-data") - continue - value = part[eoh + 4 : -2] - if not disp_params.get("name"): - gen_log.warning("multipart/form-data value missing name") - continue - name = disp_params["name"] - if disp_params.get("filename"): - ctype = headers.get("Content-Type", "application/unknown") - files.setdefault(name, []).append( - HTTPFile( # type: ignore - filename=disp_params["filename"], body=value, content_type=ctype - ) - ) - else: - arguments.setdefault(name, []).append(value) - - -def format_timestamp(ts): - """Formats a timestamp in the format used by HTTP. - - The argument may be a numeric timestamp as returned by `time.time`, - a time tuple as returned by `time.gmtime`, or a `datetime.datetime` - object. - - >>> format_timestamp(1359312200) - 'Sun, 27 Jan 2013 18:43:20 GMT' - """ - if isinstance(ts, numbers.Real): - pass - elif isinstance(ts, (tuple, time.struct_time)): - ts = calendar.timegm(ts) - elif isinstance(ts, datetime.datetime): - ts = calendar.timegm(ts.utctimetuple()) - else: - raise TypeError("unknown timestamp type: %r" % ts) - return email.utils.formatdate(ts, usegmt=True) - - -RequestStartLine = collections.namedtuple( - "RequestStartLine", ["method", "path", "version"] -) - - -def parse_request_start_line(line): - """Returns a (method, path, version) tuple for an HTTP 1.x request line. - - The response is a `collections.namedtuple`. - - >>> parse_request_start_line("GET /foo HTTP/1.1") - RequestStartLine(method='GET', path='/foo', version='HTTP/1.1') - """ - try: - method, path, version = line.split(" ") - except ValueError: - raise HTTPInputError("Malformed HTTP request line") - if not re.match(r"^HTTP/1\.[0-9]$", version): - raise HTTPInputError( - "Malformed HTTP version in HTTP Request-Line: %r" % version - ) - return RequestStartLine(method, path, version) - - -ResponseStartLine = collections.namedtuple( - "ResponseStartLine", ["version", "code", "reason"] -) - - -def parse_response_start_line(line): - """Returns a (version, code, reason) tuple for an HTTP 1.x response line. - - The response is a `collections.namedtuple`. - - >>> parse_response_start_line("HTTP/1.1 200 OK") - ResponseStartLine(version='HTTP/1.1', code=200, reason='OK') - """ - line = native_str(line) - match = re.match("(HTTP/1.[0-9]) ([0-9]+) ([^\r]*)", line) - if not match: - raise HTTPInputError("Error parsing response start line") - return ResponseStartLine(match.group(1), int(match.group(2)), match.group(3)) - - -# _parseparam and _parse_header are copied and modified from python2.7's cgi.py -# The original 2.7 version of this code did not correctly support some -# combinations of semicolons and double quotes. -# It has also been modified to support valueless parameters as seen in -# websocket extension negotiations. - - -def _parseparam(s): - while s[:1] == ";": - s = s[1:] - end = s.find(";") - while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: - end = s.find(";", end + 1) - if end < 0: - end = len(s) - f = s[:end] - yield f.strip() - s = s[end:] - - -def _parse_header(line): - """Parse a Content-type like header. - - Return the main content-type and a dictionary of options. - - """ - parts = _parseparam(";" + line) - key = next(parts) - pdict = {} - for p in parts: - i = p.find("=") - if i >= 0: - name = p[:i].strip().lower() - value = p[i + 1 :].strip() - if len(value) >= 2 and value[0] == value[-1] == '"': - value = value[1:-1] - value = value.replace("\\\\", "\\").replace('\\"', '"') - pdict[name] = value - else: - pdict[p] = None - return key, pdict - - -def _encode_header(key, pdict): - """Inverse of _parse_header. - - >>> _encode_header('permessage-deflate', - ... {'client_max_window_bits': 15, 'client_no_context_takeover': None}) - 'permessage-deflate; client_max_window_bits=15; client_no_context_takeover' - """ - if not pdict: - return key - out = [key] - # Sort the parameters just to make it easy to test. - for k, v in sorted(pdict.items()): - if v is None: - out.append(k) - else: - # TODO: quote if necessary. - out.append("%s=%s" % (k, v)) - return "; ".join(out) - - -def doctests(): - import doctest - - return doctest.DocTestSuite() - - -def split_host_and_port(netloc): - """Returns ``(host, port)`` tuple from ``netloc``. - - Returned ``port`` will be ``None`` if not present. - - .. versionadded:: 4.1 - """ - match = re.match(r"^(.+):(\d+)$", netloc) - if match: - host = match.group(1) - port = int(match.group(2)) - else: - host = netloc - port = None - return (host, port) - - -_OctalPatt = re.compile(r"\\[0-3][0-7][0-7]") -_QuotePatt = re.compile(r"[\\].") -_nulljoin = "".join - - -def _unquote_cookie(str): - """Handle double quotes and escaping in cookie values. - - This method is copied verbatim from the Python 3.5 standard - library (http.cookies._unquote) so we don't have to depend on - non-public interfaces. - """ - # If there aren't any doublequotes, - # then there can't be any special characters. See RFC 2109. - if str is None or len(str) < 2: - return str - if str[0] != '"' or str[-1] != '"': - return str - - # We have to assume that we must decode this string. - # Down to work. - - # Remove the "s - str = str[1:-1] - - # Check for special sequences. Examples: - # \012 --> \n - # \" --> " - # - i = 0 - n = len(str) - res = [] - while 0 <= i < n: - o_match = _OctalPatt.search(str, i) - q_match = _QuotePatt.search(str, i) - if not o_match and not q_match: # Neither matched - res.append(str[i:]) - break - # else: - j = k = -1 - if o_match: - j = o_match.start(0) - if q_match: - k = q_match.start(0) - if q_match and (not o_match or k < j): # QuotePatt matched - res.append(str[i:k]) - res.append(str[k + 1]) - i = k + 2 - else: # OctalPatt matched - res.append(str[i:j]) - res.append(chr(int(str[j + 1 : j + 4], 8))) - i = j + 4 - return _nulljoin(res) - - -def parse_cookie(cookie): - """Parse a ``Cookie`` HTTP header into a dict of name/value pairs. - - This function attempts to mimic browser cookie parsing behavior; - it specifically does not follow any of the cookie-related RFCs - (because browsers don't either). - - The algorithm used is identical to that used by Django version 1.9.10. - - .. versionadded:: 4.4.2 - """ - cookiedict = {} - for chunk in cookie.split(str(";")): - if str("=") in chunk: - key, val = chunk.split(str("="), 1) - else: - # Assume an empty name per - # https://bugzilla.mozilla.org/show_bug.cgi?id=169091 - key, val = str(""), chunk - key, val = key.strip(), val.strip() - if key or val: - # unquote using Python's algorithm. - cookiedict[key] = _unquote_cookie(val) - return cookiedict diff --git a/salt/ext/tornado/ioloop.py b/salt/ext/tornado/ioloop.py deleted file mode 100644 index b738805b7f8..00000000000 --- a/salt/ext/tornado/ioloop.py +++ /dev/null @@ -1,1043 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""An I/O event loop for non-blocking sockets. - -Typical applications will use a single `IOLoop` object, in the -`IOLoop.instance` singleton. The `IOLoop.start` method should usually -be called at the end of the ``main()`` function. Atypical applications may -use more than one `IOLoop`, such as one `IOLoop` per thread, or per `unittest` -case. - -In addition to I/O events, the `IOLoop` can also schedule time-based events. -`IOLoop.add_timeout` is a non-blocking alternative to `time.sleep`. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import collections -import datetime -import errno -import functools -import heapq -import itertools -import logging -import numbers -import os -import select -import sys -import threading -import time -import traceback -import math - -from salt.ext.tornado.concurrent import TracebackFuture, is_future -from salt.ext.tornado.log import app_log, gen_log -from salt.ext.tornado.platform.auto import set_close_exec, Waker -from salt.ext.tornado import stack_context -from salt.ext.tornado.util import PY3, Configurable, errno_from_exception, timedelta_to_seconds - -try: - import signal -except ImportError: - signal = None - - -if PY3: - import _thread as thread -else: - import thread - - -_POLL_TIMEOUT = 3600.0 - - -class TimeoutError(Exception): - pass - - -class IOLoop(Configurable): - """A level-triggered I/O loop. - - We use ``epoll`` (Linux) or ``kqueue`` (BSD and Mac OS X) if they - are available, or else we fall back on select(). If you are - implementing a system that needs to handle thousands of - simultaneous connections, you should use a system that supports - either ``epoll`` or ``kqueue``. - - Example usage for a simple TCP server: - - .. testcode:: - - import errno - import functools - import tornado.ioloop - import socket - - def connection_ready(sock, fd, events): - while True: - try: - connection, address = sock.accept() - except socket.error as e: - if e.args[0] not in (errno.EWOULDBLOCK, errno.EAGAIN): - raise - return - connection.setblocking(0) - handle_connection(connection, address) - - if __name__ == '__main__': - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.setblocking(0) - sock.bind(("", port)) - sock.listen(128) - - io_loop = tornado.ioloop.IOLoop.current() - callback = functools.partial(connection_ready, sock) - io_loop.add_handler(sock.fileno(), callback, io_loop.READ) - io_loop.start() - - .. testoutput:: - :hide: - - By default, a newly-constructed `IOLoop` becomes the thread's current - `IOLoop`, unless there already is a current `IOLoop`. This behavior - can be controlled with the ``make_current`` argument to the `IOLoop` - constructor: if ``make_current=True``, the new `IOLoop` will always - try to become current and it raises an error if there is already a - current instance. If ``make_current=False``, the new `IOLoop` will - not try to become current. - - .. versionchanged:: 4.2 - Added the ``make_current`` keyword argument to the `IOLoop` - constructor. - """ - # Constants from the epoll module - _EPOLLIN = 0x001 - _EPOLLPRI = 0x002 - _EPOLLOUT = 0x004 - _EPOLLERR = 0x008 - _EPOLLHUP = 0x010 - _EPOLLRDHUP = 0x2000 - _EPOLLONESHOT = (1 << 30) - _EPOLLET = (1 << 31) - - # Our events map exactly to the epoll events - NONE = 0 - READ = _EPOLLIN - WRITE = _EPOLLOUT - ERROR = _EPOLLERR | _EPOLLHUP - - # Global lock for creating global IOLoop instance - _instance_lock = threading.Lock() - - _current = threading.local() - - @staticmethod - def instance(): - """Returns a global `IOLoop` instance. - - Most applications have a single, global `IOLoop` running on the - main thread. Use this method to get this instance from - another thread. In most other cases, it is better to use `current()` - to get the current thread's `IOLoop`. - """ - if not hasattr(IOLoop, "_instance"): - with IOLoop._instance_lock: - if not hasattr(IOLoop, "_instance"): - # New instance after double check - IOLoop._instance = IOLoop() - return IOLoop._instance - - @staticmethod - def initialized(): - """Returns true if the singleton instance has been created.""" - return hasattr(IOLoop, "_instance") - - def install(self): - """Installs this `IOLoop` object as the singleton instance. - - This is normally not necessary as `instance()` will create - an `IOLoop` on demand, but you may want to call `install` to use - a custom subclass of `IOLoop`. - - When using an `IOLoop` subclass, `install` must be called prior - to creating any objects that implicitly create their own - `IOLoop` (e.g., :class:`tornado.httpclient.AsyncHTTPClient`). - """ - assert not IOLoop.initialized() - IOLoop._instance = self - - @staticmethod - def clear_instance(): - """Clear the global `IOLoop` instance. - - .. versionadded:: 4.0 - """ - if hasattr(IOLoop, "_instance"): - del IOLoop._instance - - @staticmethod - def current(instance=True): - """Returns the current thread's `IOLoop`. - - If an `IOLoop` is currently running or has been marked as - current by `make_current`, returns that instance. If there is - no current `IOLoop`, returns `IOLoop.instance()` (i.e. the - main thread's `IOLoop`, creating one if necessary) if ``instance`` - is true. - - In general you should use `IOLoop.current` as the default when - constructing an asynchronous object, and use `IOLoop.instance` - when you mean to communicate to the main thread from a different - one. - - .. versionchanged:: 4.1 - Added ``instance`` argument to control the fallback to - `IOLoop.instance()`. - """ - current = getattr(IOLoop._current, "instance", None) - if current is None and instance: - return IOLoop.instance() - return current - - def make_current(self): - """Makes this the `IOLoop` for the current thread. - - An `IOLoop` automatically becomes current for its thread - when it is started, but it is sometimes useful to call - `make_current` explicitly before starting the `IOLoop`, - so that code run at startup time can find the right - instance. - - .. versionchanged:: 4.1 - An `IOLoop` created while there is no current `IOLoop` - will automatically become current. - """ - IOLoop._current.instance = self - - @staticmethod - def clear_current(): - IOLoop._current.instance = None - - @classmethod - def configurable_base(cls): - return IOLoop - - @classmethod - def configurable_default(cls): - if hasattr(select, "epoll"): - from salt.ext.tornado.platform.epoll import EPollIOLoop - return EPollIOLoop - if hasattr(select, "kqueue"): - # Python 2.6+ on BSD or Mac - from salt.ext.tornado.platform.kqueue import KQueueIOLoop - return KQueueIOLoop - from salt.ext.tornado.platform.select import SelectIOLoop - return SelectIOLoop - - def initialize(self, make_current=None): - if make_current is None: - if IOLoop.current(instance=False) is None: - self.make_current() - elif make_current: - if IOLoop.current(instance=False) is not None: - raise RuntimeError("current IOLoop already exists") - self.make_current() - - def close(self, all_fds=False): - """Closes the `IOLoop`, freeing any resources used. - - If ``all_fds`` is true, all file descriptors registered on the - IOLoop will be closed (not just the ones created by the - `IOLoop` itself). - - Many applications will only use a single `IOLoop` that runs for the - entire lifetime of the process. In that case closing the `IOLoop` - is not necessary since everything will be cleaned up when the - process exits. `IOLoop.close` is provided mainly for scenarios - such as unit tests, which create and destroy a large number of - ``IOLoops``. - - An `IOLoop` must be completely stopped before it can be closed. This - means that `IOLoop.stop()` must be called *and* `IOLoop.start()` must - be allowed to return before attempting to call `IOLoop.close()`. - Therefore the call to `close` will usually appear just after - the call to `start` rather than near the call to `stop`. - - .. versionchanged:: 3.1 - If the `IOLoop` implementation supports non-integer objects - for "file descriptors", those objects will have their - ``close`` method when ``all_fds`` is true. - """ - raise NotImplementedError() - - def add_handler(self, fd, handler, events): - """Registers the given handler to receive the given events for ``fd``. - - The ``fd`` argument may either be an integer file descriptor or - a file-like object with a ``fileno()`` method (and optionally a - ``close()`` method, which may be called when the `IOLoop` is shut - down). - - The ``events`` argument is a bitwise or of the constants - ``IOLoop.READ``, ``IOLoop.WRITE``, and ``IOLoop.ERROR``. - - When an event occurs, ``handler(fd, events)`` will be run. - - .. versionchanged:: 4.0 - Added the ability to pass file-like objects in addition to - raw file descriptors. - """ - raise NotImplementedError() - - def update_handler(self, fd, events): - """Changes the events we listen for ``fd``. - - .. versionchanged:: 4.0 - Added the ability to pass file-like objects in addition to - raw file descriptors. - """ - raise NotImplementedError() - - def remove_handler(self, fd): - """Stop listening for events on ``fd``. - - .. versionchanged:: 4.0 - Added the ability to pass file-like objects in addition to - raw file descriptors. - """ - raise NotImplementedError() - - def set_blocking_signal_threshold(self, seconds, action): - """Sends a signal if the `IOLoop` is blocked for more than - ``s`` seconds. - - Pass ``seconds=None`` to disable. Requires Python 2.6 on a unixy - platform. - - The action parameter is a Python signal handler. Read the - documentation for the `signal` module for more information. - If ``action`` is None, the process will be killed if it is - blocked for too long. - """ - raise NotImplementedError() - - def set_blocking_log_threshold(self, seconds): - """Logs a stack trace if the `IOLoop` is blocked for more than - ``s`` seconds. - - Equivalent to ``set_blocking_signal_threshold(seconds, - self.log_stack)`` - """ - self.set_blocking_signal_threshold(seconds, self.log_stack) - - def log_stack(self, signal, frame): - """Signal handler to log the stack trace of the current thread. - - For use with `set_blocking_signal_threshold`. - """ - gen_log.warning('IOLoop blocked for %f seconds in\n%s', - self._blocking_signal_threshold, - ''.join(traceback.format_stack(frame))) - - def start(self): - """Starts the I/O loop. - - The loop will run until one of the callbacks calls `stop()`, which - will make the loop stop after the current event iteration completes. - """ - raise NotImplementedError() - - def _setup_logging(self): - """The IOLoop catches and logs exceptions, so it's - important that log output be visible. However, python's - default behavior for non-root loggers (prior to python - 3.2) is to print an unhelpful "no handlers could be - found" message rather than the actual log entry, so we - must explicitly configure logging if we've made it this - far without anything. - - This method should be called from start() in subclasses. - """ - if not any([logging.getLogger().handlers, - logging.getLogger('tornado').handlers, - logging.getLogger('tornado.application').handlers]): - logging.basicConfig() - - def stop(self): - """Stop the I/O loop. - - If the event loop is not currently running, the next call to `start()` - will return immediately. - - To use asynchronous methods from otherwise-synchronous code (such as - unit tests), you can start and stop the event loop like this:: - - ioloop = IOLoop() - async_method(ioloop=ioloop, callback=ioloop.stop) - ioloop.start() - - ``ioloop.start()`` will return after ``async_method`` has run - its callback, whether that callback was invoked before or - after ``ioloop.start``. - - Note that even after `stop` has been called, the `IOLoop` is not - completely stopped until `IOLoop.start` has also returned. - Some work that was scheduled before the call to `stop` may still - be run before the `IOLoop` shuts down. - """ - raise NotImplementedError() - - def run_sync(self, func, timeout=None): - """Starts the `IOLoop`, runs the given function, and stops the loop. - - The function must return either a yieldable object or - ``None``. If the function returns a yieldable object, the - `IOLoop` will run until the yieldable is resolved (and - `run_sync()` will return the yieldable's result). If it raises - an exception, the `IOLoop` will stop and the exception will be - re-raised to the caller. - - The keyword-only argument ``timeout`` may be used to set - a maximum duration for the function. If the timeout expires, - a `TimeoutError` is raised. - - This method is useful in conjunction with `tornado.gen.coroutine` - to allow asynchronous calls in a ``main()`` function:: - - @gen.coroutine - def main(): - # do stuff... - - if __name__ == '__main__': - IOLoop.current().run_sync(main) - - .. versionchanged:: 4.3 - Returning a non-``None``, non-yieldable value is now an error. - """ - future_cell = [None] - - def run(): - try: - result = func() - if result is not None: - from salt.ext.tornado.gen import convert_yielded - result = convert_yielded(result) - except Exception: - future_cell[0] = TracebackFuture() - future_cell[0].set_exc_info(sys.exc_info()) - else: - if is_future(result): - future_cell[0] = result - else: - future_cell[0] = TracebackFuture() - future_cell[0].set_result(result) - self.add_future(future_cell[0], lambda future: self.stop()) - self.add_callback(run) - if timeout is not None: - timeout_handle = self.add_timeout(self.time() + timeout, self.stop) - self.start() - if timeout is not None: - self.remove_timeout(timeout_handle) - if not future_cell[0].done(): - raise TimeoutError('Operation timed out after %s seconds' % timeout) - return future_cell[0].result() - - def time(self): - """Returns the current time according to the `IOLoop`'s clock. - - The return value is a floating-point number relative to an - unspecified time in the past. - - By default, the `IOLoop`'s time function is `time.time`. However, - it may be configured to use e.g. `time.monotonic` instead. - Calls to `add_timeout` that pass a number instead of a - `datetime.timedelta` should use this function to compute the - appropriate time, so they can work no matter what time function - is chosen. - """ - return time.time() - - def add_timeout(self, deadline, callback, *args, **kwargs): - """Runs the ``callback`` at the time ``deadline`` from the I/O loop. - - Returns an opaque handle that may be passed to - `remove_timeout` to cancel. - - ``deadline`` may be a number denoting a time (on the same - scale as `IOLoop.time`, normally `time.time`), or a - `datetime.timedelta` object for a deadline relative to the - current time. Since Tornado 4.0, `call_later` is a more - convenient alternative for the relative case since it does not - require a timedelta object. - - Note that it is not safe to call `add_timeout` from other threads. - Instead, you must use `add_callback` to transfer control to the - `IOLoop`'s thread, and then call `add_timeout` from there. - - Subclasses of IOLoop must implement either `add_timeout` or - `call_at`; the default implementations of each will call - the other. `call_at` is usually easier to implement, but - subclasses that wish to maintain compatibility with Tornado - versions prior to 4.0 must use `add_timeout` instead. - - .. versionchanged:: 4.0 - Now passes through ``*args`` and ``**kwargs`` to the callback. - """ - if isinstance(deadline, numbers.Real): - return self.call_at(deadline, callback, *args, **kwargs) - elif isinstance(deadline, datetime.timedelta): - return self.call_at(self.time() + timedelta_to_seconds(deadline), - callback, *args, **kwargs) - else: - raise TypeError("Unsupported deadline %r" % deadline) - - def call_later(self, delay, callback, *args, **kwargs): - """Runs the ``callback`` after ``delay`` seconds have passed. - - Returns an opaque handle that may be passed to `remove_timeout` - to cancel. Note that unlike the `asyncio` method of the same - name, the returned object does not have a ``cancel()`` method. - - See `add_timeout` for comments on thread-safety and subclassing. - - .. versionadded:: 4.0 - """ - return self.call_at(self.time() + delay, callback, *args, **kwargs) - - def call_at(self, when, callback, *args, **kwargs): - """Runs the ``callback`` at the absolute time designated by ``when``. - - ``when`` must be a number using the same reference point as - `IOLoop.time`. - - Returns an opaque handle that may be passed to `remove_timeout` - to cancel. Note that unlike the `asyncio` method of the same - name, the returned object does not have a ``cancel()`` method. - - See `add_timeout` for comments on thread-safety and subclassing. - - .. versionadded:: 4.0 - """ - return self.add_timeout(when, callback, *args, **kwargs) - - def remove_timeout(self, timeout): - """Cancels a pending timeout. - - The argument is a handle as returned by `add_timeout`. It is - safe to call `remove_timeout` even if the callback has already - been run. - """ - raise NotImplementedError() - - def add_callback(self, callback, *args, **kwargs): - """Calls the given callback on the next I/O loop iteration. - - It is safe to call this method from any thread at any time, - except from a signal handler. Note that this is the **only** - method in `IOLoop` that makes this thread-safety guarantee; all - other interaction with the `IOLoop` must be done from that - `IOLoop`'s thread. `add_callback()` may be used to transfer - control from other threads to the `IOLoop`'s thread. - - To add a callback from a signal handler, see - `add_callback_from_signal`. - """ - raise NotImplementedError() - - def add_callback_from_signal(self, callback, *args, **kwargs): - """Calls the given callback on the next I/O loop iteration. - - Safe for use from a Python signal handler; should not be used - otherwise. - - Callbacks added with this method will be run without any - `.stack_context`, to avoid picking up the context of the function - that was interrupted by the signal. - """ - raise NotImplementedError() - - def spawn_callback(self, callback, *args, **kwargs): - """Calls the given callback on the next IOLoop iteration. - - Unlike all other callback-related methods on IOLoop, - ``spawn_callback`` does not associate the callback with its caller's - ``stack_context``, so it is suitable for fire-and-forget callbacks - that should not interfere with the caller. - - .. versionadded:: 4.0 - """ - with stack_context.NullContext(): - self.add_callback(callback, *args, **kwargs) - - def add_future(self, future, callback): - """Schedules a callback on the ``IOLoop`` when the given - `.Future` is finished. - - The callback is invoked with one argument, the - `.Future`. - """ - assert is_future(future) - callback = stack_context.wrap(callback) - future.add_done_callback( - lambda future: self.add_callback(callback, future)) - - def _run_callback(self, callback): - """Runs a callback with error handling. - - For use in subclasses. - """ - try: - ret = callback() - if ret is not None: - #from salt.ext.tornado import gen - import salt.ext.tornado.gen - # Functions that return Futures typically swallow all - # exceptions and store them in the Future. If a Future - # makes it out to the IOLoop, ensure its exception (if any) - # gets logged too. - try: - ret = salt.ext.tornado.gen.convert_yielded(ret) - except salt.ext.tornado.gen.BadYieldError: - # It's not unusual for add_callback to be used with - # methods returning a non-None and non-yieldable - # result, which should just be ignored. - pass - else: - self.add_future(ret, self._discard_future_result) - except Exception: - self.handle_callback_exception(callback) - - def _discard_future_result(self, future): - """Avoid unhandled-exception warnings from spawned coroutines.""" - future.result() - - def handle_callback_exception(self, callback): - """This method is called whenever a callback run by the `IOLoop` - throws an exception. - - By default simply logs the exception as an error. Subclasses - may override this method to customize reporting of exceptions. - - The exception itself is not passed explicitly, but is available - in `sys.exc_info`. - """ - app_log.error("Exception in callback %r", callback, exc_info=True) - - def split_fd(self, fd): - """Returns an (fd, obj) pair from an ``fd`` parameter. - - We accept both raw file descriptors and file-like objects as - input to `add_handler` and related methods. When a file-like - object is passed, we must retain the object itself so we can - close it correctly when the `IOLoop` shuts down, but the - poller interfaces favor file descriptors (they will accept - file-like objects and call ``fileno()`` for you, but they - always return the descriptor itself). - - This method is provided for use by `IOLoop` subclasses and should - not generally be used by application code. - - .. versionadded:: 4.0 - """ - try: - return fd.fileno(), fd - except AttributeError: - return fd, fd - - def close_fd(self, fd): - """Utility method to close an ``fd``. - - If ``fd`` is a file-like object, we close it directly; otherwise - we use `os.close`. - - This method is provided for use by `IOLoop` subclasses (in - implementations of ``IOLoop.close(all_fds=True)`` and should - not generally be used by application code. - - .. versionadded:: 4.0 - """ - try: - try: - fd.close() - except AttributeError: - os.close(fd) - except OSError: - pass - - -class PollIOLoop(IOLoop): - """Base class for IOLoops built around a select-like function. - - For concrete implementations, see `tornado.platform.epoll.EPollIOLoop` - (Linux), `tornado.platform.kqueue.KQueueIOLoop` (BSD and Mac), or - `tornado.platform.select.SelectIOLoop` (all platforms). - """ - def initialize(self, impl, time_func=None, **kwargs): - super(PollIOLoop, self).initialize(**kwargs) - self._impl = impl - if hasattr(self._impl, 'fileno'): - set_close_exec(self._impl.fileno()) - self.time_func = time_func or time.time - self._handlers = {} - self._events = {} - self._callbacks = collections.deque() - self._timeouts = [] - self._cancellations = 0 - self._running = False - self._stopped = False - self._closing = False - self._thread_ident = None - self._blocking_signal_threshold = None - self._timeout_counter = itertools.count() - - # Create a pipe that we send bogus data to when we want to wake - # the I/O loop when it is idle - self._waker = Waker() - self.add_handler(self._waker.fileno(), - lambda fd, events: self._waker.consume(), - self.READ) - - def close(self, all_fds=False): - self._closing = True - self.remove_handler(self._waker.fileno()) - if all_fds: - for fd, handler in list(self._handlers.values()): - self.close_fd(fd) - self._waker.close() - self._impl.close() - self._callbacks = None - self._timeouts = None - - def add_handler(self, fd, handler, events): - fd, obj = self.split_fd(fd) - self._handlers[fd] = (obj, stack_context.wrap(handler)) - self._impl.register(fd, events | self.ERROR) - - def update_handler(self, fd, events): - fd, obj = self.split_fd(fd) - self._impl.modify(fd, events | self.ERROR) - - def remove_handler(self, fd): - fd, obj = self.split_fd(fd) - self._handlers.pop(fd, None) - self._events.pop(fd, None) - try: - self._impl.unregister(fd) - except Exception: - gen_log.debug("Error deleting fd from IOLoop", exc_info=True) - - def set_blocking_signal_threshold(self, seconds, action): - if not hasattr(signal, "setitimer"): - gen_log.error("set_blocking_signal_threshold requires a signal module " - "with the setitimer method") - return - self._blocking_signal_threshold = seconds - if seconds is not None: - signal.signal(signal.SIGALRM, - action if action is not None else signal.SIG_DFL) - - def start(self): - if self._running: - raise RuntimeError("IOLoop is already running") - self._setup_logging() - if self._stopped: - self._stopped = False - return - old_current = getattr(IOLoop._current, "instance", None) - IOLoop._current.instance = self - self._thread_ident = thread.get_ident() - self._running = True - - # signal.set_wakeup_fd closes a race condition in event loops: - # a signal may arrive at the beginning of select/poll/etc - # before it goes into its interruptible sleep, so the signal - # will be consumed without waking the select. The solution is - # for the (C, synchronous) signal handler to write to a pipe, - # which will then be seen by select. - # - # In python's signal handling semantics, this only matters on the - # main thread (fortunately, set_wakeup_fd only works on the main - # thread and will raise a ValueError otherwise). - # - # If someone has already set a wakeup fd, we don't want to - # disturb it. This is an issue for twisted, which does its - # SIGCHLD processing in response to its own wakeup fd being - # written to. As long as the wakeup fd is registered on the IOLoop, - # the loop will still wake up and everything should work. - old_wakeup_fd = None - if hasattr(signal, 'set_wakeup_fd') and os.name == 'posix': - # requires python 2.6+, unix. set_wakeup_fd exists but crashes - # the python process on windows. - try: - old_wakeup_fd = signal.set_wakeup_fd(self._waker.write_fileno()) - if old_wakeup_fd != -1: - # Already set, restore previous value. This is a little racy, - # but there's no clean get_wakeup_fd and in real use the - # IOLoop is just started once at the beginning. - signal.set_wakeup_fd(old_wakeup_fd) - old_wakeup_fd = None - except ValueError: - # Non-main thread, or the previous value of wakeup_fd - # is no longer valid. - old_wakeup_fd = None - - try: - while True: - # Prevent IO event starvation by delaying new callbacks - # to the next iteration of the event loop. - ncallbacks = len(self._callbacks) - - # Add any timeouts that have come due to the callback list. - # Do not run anything until we have determined which ones - # are ready, so timeouts that call add_timeout cannot - # schedule anything in this iteration. - due_timeouts = [] - if self._timeouts: - now = self.time() - while self._timeouts: - if self._timeouts[0].callback is None: - # The timeout was cancelled. Note that the - # cancellation check is repeated below for timeouts - # that are cancelled by another timeout or callback. - heapq.heappop(self._timeouts) - self._cancellations -= 1 - elif self._timeouts[0].deadline <= now: - due_timeouts.append(heapq.heappop(self._timeouts)) - else: - break - if (self._cancellations > 512 and - self._cancellations > (len(self._timeouts) >> 1)): - # Clean up the timeout queue when it gets large and it's - # more than half cancellations. - self._cancellations = 0 - self._timeouts = [x for x in self._timeouts - if x.callback is not None] - heapq.heapify(self._timeouts) - - for i in range(ncallbacks): - self._run_callback(self._callbacks.popleft()) - for timeout in due_timeouts: - if timeout.callback is not None: - self._run_callback(timeout.callback) - # Closures may be holding on to a lot of memory, so allow - # them to be freed before we go into our poll wait. - due_timeouts = timeout = None - - if self._callbacks: - # If any callbacks or timeouts called add_callback, - # we don't want to wait in poll() before we run them. - poll_timeout = 0.0 - elif self._timeouts: - # If there are any timeouts, schedule the first one. - # Use self.time() instead of 'now' to account for time - # spent running callbacks. - poll_timeout = self._timeouts[0].deadline - self.time() - poll_timeout = max(0, min(poll_timeout, _POLL_TIMEOUT)) - else: - # No timeouts and no callbacks, so use the default. - poll_timeout = _POLL_TIMEOUT - - if not self._running: - break - - if self._blocking_signal_threshold is not None: - # clear alarm so it doesn't fire while poll is waiting for - # events. - signal.setitimer(signal.ITIMER_REAL, 0, 0) - - try: - event_pairs = self._impl.poll(poll_timeout) - except Exception as e: - # Depending on python version and IOLoop implementation, - # different exception types may be thrown and there are - # two ways EINTR might be signaled: - # * e.errno == errno.EINTR - # * e.args is like (errno.EINTR, 'Interrupted system call') - if errno_from_exception(e) == errno.EINTR: - continue - else: - raise - - if self._blocking_signal_threshold is not None: - signal.setitimer(signal.ITIMER_REAL, - self._blocking_signal_threshold, 0) - - # Pop one fd at a time from the set of pending fds and run - # its handler. Since that handler may perform actions on - # other file descriptors, there may be reentrant calls to - # this IOLoop that modify self._events - self._events.update(event_pairs) - while self._events: - fd, events = self._events.popitem() - try: - fd_obj, handler_func = self._handlers[fd] - handler_func(fd_obj, events) - except (OSError, IOError) as e: - if errno_from_exception(e) == errno.EPIPE: - # Happens when the client closes the connection - pass - else: - self.handle_callback_exception(self._handlers.get(fd)) - except Exception: - self.handle_callback_exception(self._handlers.get(fd)) - fd_obj = handler_func = None - - finally: - # reset the stopped flag so another start/stop pair can be issued - self._stopped = False - if self._blocking_signal_threshold is not None: - signal.setitimer(signal.ITIMER_REAL, 0, 0) - IOLoop._current.instance = old_current - if old_wakeup_fd is not None: - signal.set_wakeup_fd(old_wakeup_fd) - - def stop(self): - self._running = False - self._stopped = True - self._waker.wake() - - def time(self): - return self.time_func() - - def call_at(self, deadline, callback, *args, **kwargs): - timeout = _Timeout( - deadline, - functools.partial(stack_context.wrap(callback), *args, **kwargs), - self) - heapq.heappush(self._timeouts, timeout) - return timeout - - def remove_timeout(self, timeout): - # Removing from a heap is complicated, so just leave the defunct - # timeout object in the queue (see discussion in - # http://docs.python.org/library/heapq.html). - # If this turns out to be a problem, we could add a garbage - # collection pass whenever there are too many dead timeouts. - timeout.callback = None - self._cancellations += 1 - - def add_callback(self, callback, *args, **kwargs): - if self._closing: - return - # Blindly insert into self._callbacks. This is safe even - # from signal handlers because deque.append is atomic. - self._callbacks.append(functools.partial( - stack_context.wrap(callback), *args, **kwargs)) - if thread.get_ident() != self._thread_ident: - # This will write one byte but Waker.consume() reads many - # at once, so it's ok to write even when not strictly - # necessary. - self._waker.wake() - else: - # If we're on the IOLoop's thread, we don't need to wake anyone. - pass - - def add_callback_from_signal(self, callback, *args, **kwargs): - with stack_context.NullContext(): - self.add_callback(callback, *args, **kwargs) - - -class _Timeout(object): - """An IOLoop timeout, a UNIX timestamp and a callback""" - - # Reduce memory overhead when there are lots of pending callbacks - __slots__ = ['deadline', 'callback', 'tdeadline'] - - def __init__(self, deadline, callback, io_loop): - if not isinstance(deadline, numbers.Real): - raise TypeError("Unsupported deadline %r" % deadline) - self.deadline = deadline - self.callback = callback - self.tdeadline = (deadline, next(io_loop._timeout_counter)) - - # Comparison methods to sort by deadline, with object id as a tiebreaker - # to guarantee a consistent ordering. The heapq module uses __le__ - # in python2.5, and __lt__ in 2.6+ (sort() and most other comparisons - # use __lt__). - def __lt__(self, other): - return self.tdeadline < other.tdeadline - - def __le__(self, other): - return self.tdeadline <= other.tdeadline - - -class PeriodicCallback(object): - """Schedules the given callback to be called periodically. - - The callback is called every ``callback_time`` milliseconds. - Note that the timeout is given in milliseconds, while most other - time-related functions in Tornado use seconds. - - If the callback runs for longer than ``callback_time`` milliseconds, - subsequent invocations will be skipped to get back on schedule. - - `start` must be called after the `PeriodicCallback` is created. - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - """ - def __init__(self, callback, callback_time, io_loop=None): - self.callback = callback - if callback_time <= 0: - raise ValueError("Periodic callback must have a positive callback_time") - self.callback_time = callback_time - self.io_loop = io_loop or IOLoop.current() - self._running = False - self._timeout = None - - def start(self): - """Starts the timer.""" - self._running = True - self._next_timeout = self.io_loop.time() - self._schedule_next() - - def stop(self): - """Stops the timer.""" - self._running = False - if self._timeout is not None: - self.io_loop.remove_timeout(self._timeout) - self._timeout = None - - def is_running(self): - """Return True if this `.PeriodicCallback` has been started. - - .. versionadded:: 4.1 - """ - return self._running - - def _run(self): - if not self._running: - return - try: - return self.callback() - except Exception: - self.io_loop.handle_callback_exception(self.callback) - finally: - self._schedule_next() - - def _schedule_next(self): - if self._running: - current_time = self.io_loop.time() - - if self._next_timeout <= current_time: - callback_time_sec = self.callback_time / 1000.0 - self._next_timeout += (math.floor((current_time - self._next_timeout) / - callback_time_sec) + 1) * callback_time_sec - - self._timeout = self.io_loop.add_timeout(self._next_timeout, self._run) diff --git a/salt/ext/tornado/iostream.py b/salt/ext/tornado/iostream.py deleted file mode 100644 index f9abf37e1fe..00000000000 --- a/salt/ext/tornado/iostream.py +++ /dev/null @@ -1,1569 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Utility classes to write to and read from non-blocking files and sockets. - -Contents: - -* `BaseIOStream`: Generic interface for reading and writing. -* `IOStream`: Implementation of BaseIOStream using non-blocking sockets. -* `SSLIOStream`: SSL-aware version of IOStream. -* `PipeIOStream`: Pipe-based IOStream implementation. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import collections -import errno -import numbers -import os -import socket -import sys -import re - -from salt.ext.tornado.concurrent import TracebackFuture -from salt.ext.tornado import ioloop -from salt.ext.tornado.log import gen_log, app_log -from salt.ext.tornado.netutil import ssl_wrap_socket, ssl_match_hostname, SSLCertificateError, _client_ssl_defaults, _server_ssl_defaults -from salt.ext.tornado import stack_context -from salt.ext.tornado.util import errno_from_exception - -try: - from salt.ext.tornado.platform.posix import _set_nonblocking -except ImportError: - _set_nonblocking = None - -try: - import ssl -except ImportError: - # ssl is not available on Google App Engine - ssl = None - -# These errnos indicate that a non-blocking operation must be retried -# at a later time. On most platforms they're the same value, but on -# some they differ. -_ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN) - -if hasattr(errno, "WSAEWOULDBLOCK"): - _ERRNO_WOULDBLOCK += (errno.WSAEWOULDBLOCK,) # type: ignore - -# These errnos indicate that a connection has been abruptly terminated. -# They should be caught and handled less noisily than other errors. -_ERRNO_CONNRESET = (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE, - errno.ETIMEDOUT) - -if hasattr(errno, "WSAECONNRESET"): - _ERRNO_CONNRESET += (errno.WSAECONNRESET, errno.WSAECONNABORTED, errno.WSAETIMEDOUT) # type: ignore - -if sys.platform == 'darwin': - # OSX appears to have a race condition that causes send(2) to return - # EPROTOTYPE if called while a socket is being torn down: - # http://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ - # Since the socket is being closed anyway, treat this as an ECONNRESET - # instead of an unexpected error. - _ERRNO_CONNRESET += (errno.EPROTOTYPE,) # type: ignore - -# More non-portable errnos: -_ERRNO_INPROGRESS = (errno.EINPROGRESS,) - -if hasattr(errno, "WSAEINPROGRESS"): - _ERRNO_INPROGRESS += (errno.WSAEINPROGRESS,) # type: ignore - -_WINDOWS = sys.platform.startswith('win') - - -class StreamClosedError(IOError): - """Exception raised by `IOStream` methods when the stream is closed. - - Note that the close callback is scheduled to run *after* other - callbacks on the stream (to allow for buffered data to be processed), - so you may see this error before you see the close callback. - - The ``real_error`` attribute contains the underlying error that caused - the stream to close (if any). - - .. versionchanged:: 4.3 - Added the ``real_error`` attribute. - """ - def __init__(self, real_error=None): - super(StreamClosedError, self).__init__('Stream is closed') - self.real_error = real_error - - -class UnsatisfiableReadError(Exception): - """Exception raised when a read cannot be satisfied. - - Raised by ``read_until`` and ``read_until_regex`` with a ``max_bytes`` - argument. - """ - pass - - -class StreamBufferFullError(Exception): - """Exception raised by `IOStream` methods when the buffer is full. - """ - - -class BaseIOStream(object): - """A utility class to write to and read from a non-blocking file or socket. - - We support a non-blocking ``write()`` and a family of ``read_*()`` methods. - All of the methods take an optional ``callback`` argument and return a - `.Future` only if no callback is given. When the operation completes, - the callback will be run or the `.Future` will resolve with the data - read (or ``None`` for ``write()``). All outstanding ``Futures`` will - resolve with a `StreamClosedError` when the stream is closed; users - of the callback interface will be notified via - `.BaseIOStream.set_close_callback` instead. - - When a stream is closed due to an error, the IOStream's ``error`` - attribute contains the exception object. - - Subclasses must implement `fileno`, `close_fd`, `write_to_fd`, - `read_from_fd`, and optionally `get_fd_error`. - """ - def __init__(self, io_loop=None, max_buffer_size=None, - read_chunk_size=None, max_write_buffer_size=None): - """`BaseIOStream` constructor. - - :arg io_loop: The `.IOLoop` to use; defaults to `.IOLoop.current`. - Deprecated since Tornado 4.1. - :arg max_buffer_size: Maximum amount of incoming data to buffer; - defaults to 100MB. - :arg read_chunk_size: Amount of data to read at one time from the - underlying transport; defaults to 64KB. - :arg max_write_buffer_size: Amount of outgoing data to buffer; - defaults to unlimited. - - .. versionchanged:: 4.0 - Add the ``max_write_buffer_size`` parameter. Changed default - ``read_chunk_size`` to 64KB. - """ - self.io_loop = io_loop or ioloop.IOLoop.current() - self.max_buffer_size = max_buffer_size or 104857600 - # A chunk size that is too close to max_buffer_size can cause - # spurious failures. - self.read_chunk_size = min(read_chunk_size or 65536, - self.max_buffer_size // 2) - self.max_write_buffer_size = max_write_buffer_size - self.error = None - self._read_buffer = bytearray() - self._read_buffer_pos = 0 - self._read_buffer_size = 0 - self._write_buffer = bytearray() - self._write_buffer_pos = 0 - self._write_buffer_size = 0 - self._write_buffer_frozen = False - self._total_write_index = 0 - self._total_write_done_index = 0 - self._pending_writes_while_frozen = [] - self._read_delimiter = None - self._read_regex = None - self._read_max_bytes = None - self._read_bytes = None - self._read_partial = False - self._read_until_close = False - self._read_callback = None - self._read_future = None - self._streaming_callback = None - self._write_callback = None - self._write_futures = collections.deque() - self._close_callback = None - self._connect_callback = None - self._connect_future = None - # _ssl_connect_future should be defined in SSLIOStream - # but it's here so we can clean it up in maybe_run_close_callback. - # TODO: refactor that so subclasses can add additional futures - # to be cancelled. - self._ssl_connect_future = None - self._connecting = False - self._state = None - self._pending_callbacks = 0 - self._closed = False - - def fileno(self): - """Returns the file descriptor for this stream.""" - raise NotImplementedError() - - def close_fd(self): - """Closes the file underlying this stream. - - ``close_fd`` is called by `BaseIOStream` and should not be called - elsewhere; other users should call `close` instead. - """ - raise NotImplementedError() - - def write_to_fd(self, data): - """Attempts to write ``data`` to the underlying file. - - Returns the number of bytes written. - """ - raise NotImplementedError() - - def read_from_fd(self): - """Attempts to read from the underlying file. - - Returns ``None`` if there was nothing to read (the socket - returned `~errno.EWOULDBLOCK` or equivalent), otherwise - returns the data. When possible, should return no more than - ``self.read_chunk_size`` bytes at a time. - """ - raise NotImplementedError() - - def get_fd_error(self): - """Returns information about any error on the underlying file. - - This method is called after the `.IOLoop` has signaled an error on the - file descriptor, and should return an Exception (such as `socket.error` - with additional information, or None if no such information is - available. - """ - return None - - def read_until_regex(self, regex, callback=None, max_bytes=None): - """Asynchronously read until we have matched the given regex. - - The result includes the data that matches the regex and anything - that came before it. If a callback is given, it will be run - with the data as an argument; if not, this method returns a - `.Future`. - - If ``max_bytes`` is not None, the connection will be closed - if more than ``max_bytes`` bytes have been read and the regex is - not satisfied. - - .. versionchanged:: 4.0 - Added the ``max_bytes`` argument. The ``callback`` argument is - now optional and a `.Future` will be returned if it is omitted. - """ - future = self._set_read_callback(callback) - self._read_regex = re.compile(regex) - self._read_max_bytes = max_bytes - try: - self._try_inline_read() - except UnsatisfiableReadError as e: - # Handle this the same way as in _handle_events. - gen_log.info("Unsatisfiable read, closing connection: %s" % e) - self.close(exc_info=True) - return future - except: - if future is not None: - # Ensure that the future doesn't log an error because its - # failure was never examined. - future.add_done_callback(lambda f: f.exception()) - raise - return future - - def read_until(self, delimiter, callback=None, max_bytes=None): - """Asynchronously read until we have found the given delimiter. - - The result includes all the data read including the delimiter. - If a callback is given, it will be run with the data as an argument; - if not, this method returns a `.Future`. - - If ``max_bytes`` is not None, the connection will be closed - if more than ``max_bytes`` bytes have been read and the delimiter - is not found. - - .. versionchanged:: 4.0 - Added the ``max_bytes`` argument. The ``callback`` argument is - now optional and a `.Future` will be returned if it is omitted. - """ - future = self._set_read_callback(callback) - self._read_delimiter = delimiter - self._read_max_bytes = max_bytes - try: - self._try_inline_read() - except UnsatisfiableReadError as e: - # Handle this the same way as in _handle_events. - gen_log.info("Unsatisfiable read, closing connection: %s" % e) - self.close(exc_info=True) - return future - except: - if future is not None: - future.add_done_callback(lambda f: f.exception()) - raise - return future - - def read_bytes(self, num_bytes, callback=None, streaming_callback=None, - partial=False): - """Asynchronously read a number of bytes. - - If a ``streaming_callback`` is given, it will be called with chunks - of data as they become available, and the final result will be empty. - Otherwise, the result is all the data that was read. - If a callback is given, it will be run with the data as an argument; - if not, this method returns a `.Future`. - - If ``partial`` is true, the callback is run as soon as we have - any bytes to return (but never more than ``num_bytes``) - - .. versionchanged:: 4.0 - Added the ``partial`` argument. The callback argument is now - optional and a `.Future` will be returned if it is omitted. - """ - future = self._set_read_callback(callback) - assert isinstance(num_bytes, numbers.Integral) - self._read_bytes = num_bytes - self._read_partial = partial - self._streaming_callback = stack_context.wrap(streaming_callback) - try: - self._try_inline_read() - except: - if future is not None: - future.add_done_callback(lambda f: f.exception()) - raise - return future - - def read_until_close(self, callback=None, streaming_callback=None): - """Asynchronously reads all data from the socket until it is closed. - - If a ``streaming_callback`` is given, it will be called with chunks - of data as they become available, and the final result will be empty. - Otherwise, the result is all the data that was read. - If a callback is given, it will be run with the data as an argument; - if not, this method returns a `.Future`. - - Note that if a ``streaming_callback`` is used, data will be - read from the socket as quickly as it becomes available; there - is no way to apply backpressure or cancel the reads. If flow - control or cancellation are desired, use a loop with - `read_bytes(partial=True) <.read_bytes>` instead. - - .. versionchanged:: 4.0 - The callback argument is now optional and a `.Future` will - be returned if it is omitted. - - """ - future = self._set_read_callback(callback) - self._streaming_callback = stack_context.wrap(streaming_callback) - if self.closed(): - if self._streaming_callback is not None: - self._run_read_callback(self._read_buffer_size, True) - self._run_read_callback(self._read_buffer_size, False) - return future - self._read_until_close = True - try: - self._try_inline_read() - except: - if future is not None: - future.add_done_callback(lambda f: f.exception()) - raise - return future - - def write(self, data, callback=None): - """Asynchronously write the given data to this stream. - - If ``callback`` is given, we call it when all of the buffered write - data has been successfully written to the stream. If there was - previously buffered write data and an old write callback, that - callback is simply overwritten with this new callback. - - If no ``callback`` is given, this method returns a `.Future` that - resolves (with a result of ``None``) when the write has been - completed. - - The ``data`` argument may be of type `bytes` or `memoryview`. - - .. versionchanged:: 4.0 - Now returns a `.Future` if no callback is given. - - .. versionchanged:: 4.5 - Added support for `memoryview` arguments. - """ - self._check_closed() - if data: - if (self.max_write_buffer_size is not None and - self._write_buffer_size + len(data) > self.max_write_buffer_size): - raise StreamBufferFullError("Reached maximum write buffer size") - if self._write_buffer_frozen: - self._pending_writes_while_frozen.append(data) - else: - self._write_buffer += data - self._write_buffer_size += len(data) - self._total_write_index += len(data) - if callback is not None: - self._write_callback = stack_context.wrap(callback) - future = None - else: - future = TracebackFuture() - future.add_done_callback(lambda f: f.exception()) - self._write_futures.append((self._total_write_index, future)) - if not self._connecting: - self._handle_write() - if self._write_buffer_size: - self._add_io_state(self.io_loop.WRITE) - self._maybe_add_error_listener() - return future - - def set_close_callback(self, callback): - """Call the given callback when the stream is closed. - - This is not necessary for applications that use the `.Future` - interface; all outstanding ``Futures`` will resolve with a - `StreamClosedError` when the stream is closed. - """ - self._close_callback = stack_context.wrap(callback) - self._maybe_add_error_listener() - - def close(self, exc_info=False): - """Close this stream. - - If ``exc_info`` is true, set the ``error`` attribute to the current - exception from `sys.exc_info` (or if ``exc_info`` is a tuple, - use that instead of `sys.exc_info`). - """ - if not self.closed(): - if exc_info: - if not isinstance(exc_info, tuple): - exc_info = sys.exc_info() - if any(exc_info): - self.error = exc_info[1] - if self._read_until_close: - if (self._streaming_callback is not None and - self._read_buffer_size): - self._run_read_callback(self._read_buffer_size, True) - self._read_until_close = False - self._run_read_callback(self._read_buffer_size, False) - if self._state is not None: - self.io_loop.remove_handler(self.fileno()) - self._state = None - self.close_fd() - self._closed = True - self._maybe_run_close_callback() - - def _maybe_run_close_callback(self): - # If there are pending callbacks, don't run the close callback - # until they're done (see _maybe_add_error_handler) - if self.closed() and self._pending_callbacks == 0: - futures = [] - if self._read_future is not None: - futures.append(self._read_future) - self._read_future = None - futures += [future for _, future in self._write_futures] - self._write_futures.clear() - if self._connect_future is not None: - futures.append(self._connect_future) - self._connect_future = None - if self._ssl_connect_future is not None: - futures.append(self._ssl_connect_future) - self._ssl_connect_future = None - for future in futures: - future.set_exception(StreamClosedError(real_error=self.error)) - if self._close_callback is not None: - cb = self._close_callback - self._close_callback = None - self._run_callback(cb) - # Delete any unfinished callbacks to break up reference cycles. - self._read_callback = self._write_callback = None - # Clear the buffers so they can be cleared immediately even - # if the IOStream object is kept alive by a reference cycle. - # TODO: Clear the read buffer too; it currently breaks some tests. - self._write_buffer = None - self._write_buffer_size = 0 - - def reading(self): - """Returns true if we are currently reading from the stream.""" - return self._read_callback is not None or self._read_future is not None - - def writing(self): - """Returns true if we are currently writing to the stream.""" - return self._write_buffer_size > 0 - - def closed(self): - """Returns true if the stream has been closed.""" - return self._closed - - def set_nodelay(self, value): - """Sets the no-delay flag for this stream. - - By default, data written to TCP streams may be held for a time - to make the most efficient use of bandwidth (according to - Nagle's algorithm). The no-delay flag requests that data be - written as soon as possible, even if doing so would consume - additional bandwidth. - - This flag is currently defined only for TCP-based ``IOStreams``. - - .. versionadded:: 3.1 - """ - pass - - def _handle_events(self, fd, events): - if self.closed(): - gen_log.warning("Got events for closed stream %s", fd) - return - try: - if self._connecting: - # Most IOLoops will report a write failed connect - # with the WRITE event, but SelectIOLoop reports a - # READ as well so we must check for connecting before - # either. - self._handle_connect() - if self.closed(): - return - if events & self.io_loop.READ: - self._handle_read() - if self.closed(): - return - if events & self.io_loop.WRITE: - self._handle_write() - if self.closed(): - return - if events & self.io_loop.ERROR: - self.error = self.get_fd_error() - # We may have queued up a user callback in _handle_read or - # _handle_write, so don't close the IOStream until those - # callbacks have had a chance to run. - self.io_loop.add_callback(self.close) - return - state = self.io_loop.ERROR - if self.reading(): - state |= self.io_loop.READ - if self.writing(): - state |= self.io_loop.WRITE - if state == self.io_loop.ERROR and self._read_buffer_size == 0: - # If the connection is idle, listen for reads too so - # we can tell if the connection is closed. If there is - # data in the read buffer we won't run the close callback - # yet anyway, so we don't need to listen in this case. - state |= self.io_loop.READ - if state != self._state: - assert self._state is not None, \ - "shouldn't happen: _handle_events without self._state" - self._state = state - self.io_loop.update_handler(self.fileno(), self._state) - except UnsatisfiableReadError as e: - gen_log.info("Unsatisfiable read, closing connection: %s" % e) - self.close(exc_info=True) - except Exception: - gen_log.error("Uncaught exception, closing connection.", - exc_info=True) - self.close(exc_info=True) - raise - - def _run_callback(self, callback, *args): - def wrapper(): - self._pending_callbacks -= 1 - try: - return callback(*args) - except Exception: - app_log.error("Uncaught exception, closing connection.", - exc_info=True) - # Close the socket on an uncaught exception from a user callback - # (It would eventually get closed when the socket object is - # gc'd, but we don't want to rely on gc happening before we - # run out of file descriptors) - self.close(exc_info=True) - # Re-raise the exception so that IOLoop.handle_callback_exception - # can see it and log the error - raise - finally: - self._maybe_add_error_listener() - # We schedule callbacks to be run on the next IOLoop iteration - # rather than running them directly for several reasons: - # * Prevents unbounded stack growth when a callback calls an - # IOLoop operation that immediately runs another callback - # * Provides a predictable execution context for e.g. - # non-reentrant mutexes - # * Ensures that the try/except in wrapper() is run outside - # of the application's StackContexts - with stack_context.NullContext(): - # stack_context was already captured in callback, we don't need to - # capture it again for IOStream's wrapper. This is especially - # important if the callback was pre-wrapped before entry to - # IOStream (as in HTTPConnection._header_callback), as we could - # capture and leak the wrong context here. - self._pending_callbacks += 1 - self.io_loop.add_callback(wrapper) - - def _read_to_buffer_loop(self): - # This method is called from _handle_read and _try_inline_read. - try: - if self._read_bytes is not None: - target_bytes = self._read_bytes - elif self._read_max_bytes is not None: - target_bytes = self._read_max_bytes - elif self.reading(): - # For read_until without max_bytes, or - # read_until_close, read as much as we can before - # scanning for the delimiter. - target_bytes = None - else: - target_bytes = 0 - next_find_pos = 0 - # Pretend to have a pending callback so that an EOF in - # _read_to_buffer doesn't trigger an immediate close - # callback. At the end of this method we'll either - # establish a real pending callback via - # _read_from_buffer or run the close callback. - # - # We need two try statements here so that - # pending_callbacks is decremented before the `except` - # clause below (which calls `close` and does need to - # trigger the callback) - self._pending_callbacks += 1 - while not self.closed(): - # Read from the socket until we get EWOULDBLOCK or equivalent. - # SSL sockets do some internal buffering, and if the data is - # sitting in the SSL object's buffer select() and friends - # can't see it; the only way to find out if it's there is to - # try to read it. - if self._read_to_buffer() == 0: - break - - self._run_streaming_callback() - - # If we've read all the bytes we can use, break out of - # this loop. We can't just call read_from_buffer here - # because of subtle interactions with the - # pending_callback and error_listener mechanisms. - # - # If we've reached target_bytes, we know we're done. - if (target_bytes is not None and - self._read_buffer_size >= target_bytes): - break - - # Otherwise, we need to call the more expensive find_read_pos. - # It's inefficient to do this on every read, so instead - # do it on the first read and whenever the read buffer - # size has doubled. - if self._read_buffer_size >= next_find_pos: - pos = self._find_read_pos() - if pos is not None: - return pos - next_find_pos = self._read_buffer_size * 2 - return self._find_read_pos() - finally: - self._pending_callbacks -= 1 - - def _handle_read(self): - try: - pos = self._read_to_buffer_loop() - except UnsatisfiableReadError: - raise - except Exception as e: - gen_log.warning("error on read: %s" % e) - self.close(exc_info=True) - return - if pos is not None: - self._read_from_buffer(pos) - return - else: - self._maybe_run_close_callback() - - def _set_read_callback(self, callback): - assert self._read_callback is None, "Already reading" - assert self._read_future is None, "Already reading" - if callback is not None: - self._read_callback = stack_context.wrap(callback) - else: - self._read_future = TracebackFuture() - return self._read_future - - def _run_read_callback(self, size, streaming): - if streaming: - callback = self._streaming_callback - else: - callback = self._read_callback - self._read_callback = self._streaming_callback = None - if self._read_future is not None: - assert callback is None - future = self._read_future - self._read_future = None - future.set_result(self._consume(size)) - if callback is not None: - assert (self._read_future is None) or streaming - self._run_callback(callback, self._consume(size)) - else: - # If we scheduled a callback, we will add the error listener - # afterwards. If we didn't, we have to do it now. - self._maybe_add_error_listener() - - def _try_inline_read(self): - """Attempt to complete the current read operation from buffered data. - - If the read can be completed without blocking, schedules the - read callback on the next IOLoop iteration; otherwise starts - listening for reads on the socket. - """ - # See if we've already got the data from a previous read - self._run_streaming_callback() - pos = self._find_read_pos() - if pos is not None: - self._read_from_buffer(pos) - return - self._check_closed() - try: - pos = self._read_to_buffer_loop() - except Exception: - # If there was an in _read_to_buffer, we called close() already, - # but couldn't run the close callback because of _pending_callbacks. - # Before we escape from this function, run the close callback if - # applicable. - self._maybe_run_close_callback() - raise - if pos is not None: - self._read_from_buffer(pos) - return - # We couldn't satisfy the read inline, so either close the stream - # or listen for new data. - if self.closed(): - self._maybe_run_close_callback() - else: - self._add_io_state(ioloop.IOLoop.READ) - - def _read_to_buffer(self): - """Reads from the socket and appends the result to the read buffer. - - Returns the number of bytes read. Returns 0 if there is nothing - to read (i.e. the read returns EWOULDBLOCK or equivalent). On - error closes the socket and raises an exception. - """ - while True: - try: - chunk = self.read_from_fd() - except (socket.error, IOError, OSError) as e: - if errno_from_exception(e) == errno.EINTR: - continue - # ssl.SSLError is a subclass of socket.error - if self._is_connreset(e): - # Treat ECONNRESET as a connection close rather than - # an error to minimize log spam (the exception will - # be available on self.error for apps that care). - self.close(exc_info=True) - return - self.close(exc_info=True) - raise - break - if chunk is None: - return 0 - self._read_buffer += chunk - self._read_buffer_size += len(chunk) - if self._read_buffer_size > self.max_buffer_size: - gen_log.error("Reached maximum read buffer size") - self.close() - raise StreamBufferFullError("Reached maximum read buffer size") - return len(chunk) - - def _run_streaming_callback(self): - if self._streaming_callback is not None and self._read_buffer_size: - bytes_to_consume = self._read_buffer_size - if self._read_bytes is not None: - bytes_to_consume = min(self._read_bytes, bytes_to_consume) - self._read_bytes -= bytes_to_consume - self._run_read_callback(bytes_to_consume, True) - - def _read_from_buffer(self, pos): - """Attempts to complete the currently-pending read from the buffer. - - The argument is either a position in the read buffer or None, - as returned by _find_read_pos. - """ - self._read_bytes = self._read_delimiter = self._read_regex = None - self._read_partial = False - self._run_read_callback(pos, False) - - def _find_read_pos(self): - """Attempts to find a position in the read buffer that satisfies - the currently-pending read. - - Returns a position in the buffer if the current read can be satisfied, - or None if it cannot. - """ - if (self._read_bytes is not None and - (self._read_buffer_size >= self._read_bytes or - (self._read_partial and self._read_buffer_size > 0))): - num_bytes = min(self._read_bytes, self._read_buffer_size) - return num_bytes - elif self._read_delimiter is not None: - # Multi-byte delimiters (e.g. '\r\n') may straddle two - # chunks in the read buffer, so we can't easily find them - # without collapsing the buffer. However, since protocols - # using delimited reads (as opposed to reads of a known - # length) tend to be "line" oriented, the delimiter is likely - # to be in the first few chunks. Merge the buffer gradually - # since large merges are relatively expensive and get undone in - # _consume(). - if self._read_buffer: - loc = self._read_buffer.find(self._read_delimiter, - self._read_buffer_pos) - if loc != -1: - loc -= self._read_buffer_pos - delimiter_len = len(self._read_delimiter) - self._check_max_bytes(self._read_delimiter, - loc + delimiter_len) - return loc + delimiter_len - self._check_max_bytes(self._read_delimiter, - self._read_buffer_size) - elif self._read_regex is not None: - if self._read_buffer: - m = self._read_regex.search(self._read_buffer, - self._read_buffer_pos) - if m is not None: - loc = m.end() - self._read_buffer_pos - self._check_max_bytes(self._read_regex, loc) - return loc - self._check_max_bytes(self._read_regex, self._read_buffer_size) - return None - - def _check_max_bytes(self, delimiter, size): - if (self._read_max_bytes is not None and - size > self._read_max_bytes): - raise UnsatisfiableReadError( - "delimiter %r not found within %d bytes" % ( - delimiter, self._read_max_bytes)) - - def _freeze_write_buffer(self, size): - self._write_buffer_frozen = size - - def _unfreeze_write_buffer(self): - self._write_buffer_frozen = False - self._write_buffer += b''.join(self._pending_writes_while_frozen) - self._write_buffer_size += sum(map(len, self._pending_writes_while_frozen)) - self._pending_writes_while_frozen[:] = [] - - def _got_empty_write(self, size): - """ - Called when a non-blocking write() failed writing anything. - Can be overridden in subclasses. - """ - - def _handle_write(self): - while self._write_buffer_size: - assert self._write_buffer_size >= 0 - try: - start = self._write_buffer_pos - if self._write_buffer_frozen: - size = self._write_buffer_frozen - elif _WINDOWS: - # On windows, socket.send blows up if given a - # write buffer that's too large, instead of just - # returning the number of bytes it was able to - # process. Therefore we must not call socket.send - # with more than 128KB at a time. - size = 128 * 1024 - else: - size = self._write_buffer_size - num_bytes = self.write_to_fd( - memoryview(self._write_buffer)[start:start + size]) - if num_bytes == 0: - self._got_empty_write(size) - break - self._write_buffer_pos += num_bytes - self._write_buffer_size -= num_bytes - # Amortized O(1) shrink - # (this heuristic is implemented natively in Python 3.4+ - # but is replicated here for Python 2) - if self._write_buffer_pos > self._write_buffer_size: - del self._write_buffer[:self._write_buffer_pos] - self._write_buffer_pos = 0 - if self._write_buffer_frozen: - self._unfreeze_write_buffer() - self._total_write_done_index += num_bytes - except (socket.error, IOError, OSError) as e: - if e.args[0] in _ERRNO_WOULDBLOCK: - self._got_empty_write(size) - break - else: - if not self._is_connreset(e): - # Broken pipe errors are usually caused by connection - # reset, and its better to not log EPIPE errors to - # minimize log spam - gen_log.warning("Write error on %s: %s", - self.fileno(), e) - self.close(exc_info=True) - return - - while self._write_futures: - index, future = self._write_futures[0] - if index > self._total_write_done_index: - break - self._write_futures.popleft() - future.set_result(None) - - if not self._write_buffer_size: - if self._write_callback: - callback = self._write_callback - self._write_callback = None - self._run_callback(callback) - - def _consume(self, loc): - # Consume loc bytes from the read buffer and return them - if loc == 0: - return b"" - assert loc <= self._read_buffer_size - # Slice the bytearray buffer into bytes, without intermediate copying - b = (memoryview(self._read_buffer) - [self._read_buffer_pos:self._read_buffer_pos + loc] - ).tobytes() - self._read_buffer_pos += loc - self._read_buffer_size -= loc - # Amortized O(1) shrink - # (this heuristic is implemented natively in Python 3.4+ - # but is replicated here for Python 2) - if self._read_buffer_pos > self._read_buffer_size: - del self._read_buffer[:self._read_buffer_pos] - self._read_buffer_pos = 0 - return b - - def _check_closed(self): - if self.closed(): - raise StreamClosedError(real_error=self.error) - - def _maybe_add_error_listener(self): - # This method is part of an optimization: to detect a connection that - # is closed when we're not actively reading or writing, we must listen - # for read events. However, it is inefficient to do this when the - # connection is first established because we are going to read or write - # immediately anyway. Instead, we insert checks at various times to - # see if the connection is idle and add the read listener then. - if self._pending_callbacks != 0: - return - if self._state is None or self._state == ioloop.IOLoop.ERROR: - if self.closed(): - self._maybe_run_close_callback() - elif (self._read_buffer_size == 0 and - self._close_callback is not None): - self._add_io_state(ioloop.IOLoop.READ) - - def _add_io_state(self, state): - """Adds `state` (IOLoop.{READ,WRITE} flags) to our event handler. - - Implementation notes: Reads and writes have a fast path and a - slow path. The fast path reads synchronously from socket - buffers, while the slow path uses `_add_io_state` to schedule - an IOLoop callback. Note that in both cases, the callback is - run asynchronously with `_run_callback`. - - To detect closed connections, we must have called - `_add_io_state` at some point, but we want to delay this as - much as possible so we don't have to set an `IOLoop.ERROR` - listener that will be overwritten by the next slow-path - operation. As long as there are callbacks scheduled for - fast-path ops, those callbacks may do more reads. - If a sequence of fast-path ops do not end in a slow-path op, - (e.g. for an @asynchronous long-poll request), we must add - the error handler. This is done in `_run_callback` and `write` - (since the write callback is optional so we can have a - fast-path write with no `_run_callback`) - """ - if self.closed(): - # connection has been closed, so there can be no future events - return - if self._state is None: - self._state = ioloop.IOLoop.ERROR | state - with stack_context.NullContext(): - self.io_loop.add_handler( - self.fileno(), self._handle_events, self._state) - elif not self._state & state: - self._state = self._state | state - self.io_loop.update_handler(self.fileno(), self._state) - - def _is_connreset(self, exc): - """Return true if exc is ECONNRESET or equivalent. - - May be overridden in subclasses. - """ - return (isinstance(exc, (socket.error, IOError)) and - errno_from_exception(exc) in _ERRNO_CONNRESET) - - -class IOStream(BaseIOStream): - r"""Socket-based `IOStream` implementation. - - This class supports the read and write methods from `BaseIOStream` - plus a `connect` method. - - The ``socket`` parameter may either be connected or unconnected. - For server operations the socket is the result of calling - `socket.accept `. For client operations the - socket is created with `socket.socket`, and may either be - connected before passing it to the `IOStream` or connected with - `IOStream.connect`. - - A very simple (and broken) HTTP client using this class: - - .. testcode:: - - import tornado.ioloop - import tornado.iostream - import socket - - def send_request(): - stream.write(b"GET / HTTP/1.0\r\nHost: friendfeed.com\r\n\r\n") - stream.read_until(b"\r\n\r\n", on_headers) - - def on_headers(data): - headers = {} - for line in data.split(b"\r\n"): - parts = line.split(b":") - if len(parts) == 2: - headers[parts[0].strip()] = parts[1].strip() - stream.read_bytes(int(headers[b"Content-Length"]), on_body) - - def on_body(data): - print(data) - stream.close() - tornado.ioloop.IOLoop.current().stop() - - if __name__ == '__main__': - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) - stream = tornado.iostream.IOStream(s) - stream.connect(("friendfeed.com", 80), send_request) - tornado.ioloop.IOLoop.current().start() - - .. testoutput:: - :hide: - - """ - def __init__(self, socket, *args, **kwargs): - self.socket = socket - self.socket.setblocking(False) - super(IOStream, self).__init__(*args, **kwargs) - - def fileno(self): - return self.socket - - def close_fd(self): - self.socket.close() - self.socket = None - - def get_fd_error(self): - errno = self.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_ERROR) - return socket.error(errno, os.strerror(errno)) - - def read_from_fd(self): - try: - chunk = self.socket.recv(self.read_chunk_size) - except socket.error as e: - if e.args[0] in _ERRNO_WOULDBLOCK: - return None - else: - raise - if not chunk: - self.close() - return None - return chunk - - def write_to_fd(self, data): - try: - return self.socket.send(data) - finally: - # Avoid keeping to data, which can be a memoryview. - # See https://github.com/tornadoweb/tornado/pull/2008 - del data - - def connect(self, address, callback=None, server_hostname=None): - """Connects the socket to a remote address without blocking. - - May only be called if the socket passed to the constructor was - not previously connected. The address parameter is in the - same format as for `socket.connect ` for - the type of socket passed to the IOStream constructor, - e.g. an ``(ip, port)`` tuple. Hostnames are accepted here, - but will be resolved synchronously and block the IOLoop. - If you have a hostname instead of an IP address, the `.TCPClient` - class is recommended instead of calling this method directly. - `.TCPClient` will do asynchronous DNS resolution and handle - both IPv4 and IPv6. - - If ``callback`` is specified, it will be called with no - arguments when the connection is completed; if not this method - returns a `.Future` (whose result after a successful - connection will be the stream itself). - - In SSL mode, the ``server_hostname`` parameter will be used - for certificate validation (unless disabled in the - ``ssl_options``) and SNI (if supported; requires Python - 2.7.9+). - - Note that it is safe to call `IOStream.write - ` while the connection is pending, in - which case the data will be written as soon as the connection - is ready. Calling `IOStream` read methods before the socket is - connected works on some platforms but is non-portable. - - .. versionchanged:: 4.0 - If no callback is given, returns a `.Future`. - - .. versionchanged:: 4.2 - SSL certificates are validated by default; pass - ``ssl_options=dict(cert_reqs=ssl.CERT_NONE)`` or a - suitably-configured `ssl.SSLContext` to the - `SSLIOStream` constructor to disable. - """ - self._connecting = True - if callback is not None: - self._connect_callback = stack_context.wrap(callback) - future = None - else: - future = self._connect_future = TracebackFuture() - try: - self.socket.connect(address) - except socket.error as e: - # In non-blocking mode we expect connect() to raise an - # exception with EINPROGRESS or EWOULDBLOCK. - # - # On freebsd, other errors such as ECONNREFUSED may be - # returned immediately when attempting to connect to - # localhost, so handle them the same way as an error - # reported later in _handle_connect. - if (errno_from_exception(e) not in _ERRNO_INPROGRESS and - errno_from_exception(e) not in _ERRNO_WOULDBLOCK): - if future is None: - gen_log.warning("Connect error on fd %s: %s", - self.socket.fileno(), e) - self.close(exc_info=True) - return future - self._add_io_state(self.io_loop.WRITE) - return future - - def start_tls(self, server_side, ssl_options=None, server_hostname=None): - """Convert this `IOStream` to an `SSLIOStream`. - - This enables protocols that begin in clear-text mode and - switch to SSL after some initial negotiation (such as the - ``STARTTLS`` extension to SMTP and IMAP). - - This method cannot be used if there are outstanding reads - or writes on the stream, or if there is any data in the - IOStream's buffer (data in the operating system's socket - buffer is allowed). This means it must generally be used - immediately after reading or writing the last clear-text - data. It can also be used immediately after connecting, - before any reads or writes. - - The ``ssl_options`` argument may be either an `ssl.SSLContext` - object or a dictionary of keyword arguments for the - `ssl.wrap_socket` function. The ``server_hostname`` argument - will be used for certificate validation unless disabled - in the ``ssl_options``. - - This method returns a `.Future` whose result is the new - `SSLIOStream`. After this method has been called, - any other operation on the original stream is undefined. - - If a close callback is defined on this stream, it will be - transferred to the new stream. - - .. versionadded:: 4.0 - - .. versionchanged:: 4.2 - SSL certificates are validated by default; pass - ``ssl_options=dict(cert_reqs=ssl.CERT_NONE)`` or a - suitably-configured `ssl.SSLContext` to disable. - """ - if (self._read_callback or self._read_future or - self._write_callback or self._write_futures or - self._connect_callback or self._connect_future or - self._pending_callbacks or self._closed or - self._read_buffer or self._write_buffer): - raise ValueError("IOStream is not idle; cannot convert to SSL") - if ssl_options is None: - if server_side: - ssl_options = _server_ssl_defaults - else: - ssl_options = _client_ssl_defaults - - socket = self.socket - self.io_loop.remove_handler(socket) - self.socket = None - socket = ssl_wrap_socket(socket, ssl_options, - server_hostname=server_hostname, - server_side=server_side, - do_handshake_on_connect=False) - orig_close_callback = self._close_callback - self._close_callback = None - - future = TracebackFuture() - ssl_stream = SSLIOStream(socket, ssl_options=ssl_options, - io_loop=self.io_loop) - # Wrap the original close callback so we can fail our Future as well. - # If we had an "unwrap" counterpart to this method we would need - # to restore the original callback after our Future resolves - # so that repeated wrap/unwrap calls don't build up layers. - - def close_callback(): - if not future.done(): - # Note that unlike most Futures returned by IOStream, - # this one passes the underlying error through directly - # instead of wrapping everything in a StreamClosedError - # with a real_error attribute. This is because once the - # connection is established it's more helpful to raise - # the SSLError directly than to hide it behind a - # StreamClosedError (and the client is expecting SSL - # issues rather than network issues since this method is - # named start_tls). - future.set_exception(ssl_stream.error or StreamClosedError()) - if orig_close_callback is not None: - orig_close_callback() - ssl_stream.set_close_callback(close_callback) - ssl_stream._ssl_connect_callback = lambda: future.set_result(ssl_stream) - ssl_stream.max_buffer_size = self.max_buffer_size - ssl_stream.read_chunk_size = self.read_chunk_size - return future - - def _handle_connect(self): - err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - if err != 0: - self.error = socket.error(err, os.strerror(err)) - # IOLoop implementations may vary: some of them return - # an error state before the socket becomes writable, so - # in that case a connection failure would be handled by the - # error path in _handle_events instead of here. - if self._connect_future is None: - gen_log.warning("Connect error on fd %s: %s", - self.socket.fileno(), errno.errorcode[err]) - self.close() - return - if self._connect_callback is not None: - callback = self._connect_callback - self._connect_callback = None - self._run_callback(callback) - if self._connect_future is not None: - future = self._connect_future - self._connect_future = None - future.set_result(self) - self._connecting = False - - def set_nodelay(self, value): - if (self.socket is not None and - self.socket.family in (socket.AF_INET, socket.AF_INET6)): - try: - self.socket.setsockopt(socket.IPPROTO_TCP, - socket.TCP_NODELAY, 1 if value else 0) - except socket.error as e: - # Sometimes setsockopt will fail if the socket is closed - # at the wrong time. This can happen with HTTPServer - # resetting the value to false between requests. - if e.errno != errno.EINVAL and not self._is_connreset(e): - raise - - -class SSLIOStream(IOStream): - """A utility class to write to and read from a non-blocking SSL socket. - - If the socket passed to the constructor is already connected, - it should be wrapped with:: - - ssl.wrap_socket(sock, do_handshake_on_connect=False, **kwargs) - - before constructing the `SSLIOStream`. Unconnected sockets will be - wrapped when `IOStream.connect` is finished. - """ - def __init__(self, *args, **kwargs): - """The ``ssl_options`` keyword argument may either be an - `ssl.SSLContext` object or a dictionary of keywords arguments - for `ssl.wrap_socket` - """ - self._ssl_options = kwargs.pop('ssl_options', _client_ssl_defaults) - super(SSLIOStream, self).__init__(*args, **kwargs) - self._ssl_accepting = True - self._handshake_reading = False - self._handshake_writing = False - self._ssl_connect_callback = None - self._server_hostname = None - - # If the socket is already connected, attempt to start the handshake. - try: - self.socket.getpeername() - except socket.error: - pass - else: - # Indirectly start the handshake, which will run on the next - # IOLoop iteration and then the real IO state will be set in - # _handle_events. - self._add_io_state(self.io_loop.WRITE) - - def reading(self): - return self._handshake_reading or super(SSLIOStream, self).reading() - - def writing(self): - return self._handshake_writing or super(SSLIOStream, self).writing() - - def _got_empty_write(self, size): - # With OpenSSL, if we couldn't write the entire buffer, - # the very same string object must be used on the - # next call to send. Therefore we suppress - # merging the write buffer after an incomplete send. - # A cleaner solution would be to set - # SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER, but this is - # not yet accessible from python - # (http://bugs.python.org/issue8240) - self._freeze_write_buffer(size) - - def _do_ssl_handshake(self): - # Based on code from test_ssl.py in the python stdlib - try: - self._handshake_reading = False - self._handshake_writing = False - self.socket.do_handshake() - except ssl.SSLError as err: - if err.args[0] == ssl.SSL_ERROR_WANT_READ: - self._handshake_reading = True - return - elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE: - self._handshake_writing = True - return - elif err.args[0] in (ssl.SSL_ERROR_EOF, - ssl.SSL_ERROR_ZERO_RETURN): - return self.close(exc_info=True) - elif err.args[0] == ssl.SSL_ERROR_SSL: - try: - peer = self.socket.getpeername() - except Exception: - peer = '(not connected)' - gen_log.warning("SSL Error on %s %s: %s", - self.socket.fileno(), peer, err) - return self.close(exc_info=True) - raise - except socket.error as err: - # Some port scans (e.g. nmap in -sT mode) have been known - # to cause do_handshake to raise EBADF and ENOTCONN, so make - # those errors quiet as well. - # https://groups.google.com/forum/?fromgroups#!topic/python-tornado/ApucKJat1_0 - if (self._is_connreset(err) or - err.args[0] in (errno.EBADF, errno.ENOTCONN)): - return self.close(exc_info=True) - raise - except AttributeError: - # On Linux, if the connection was reset before the call to - # wrap_socket, do_handshake will fail with an - # AttributeError. - return self.close(exc_info=True) - else: - self._ssl_accepting = False - if not self._verify_cert(self.socket.getpeercert()): - self.close() - return - self._run_ssl_connect_callback() - - def _run_ssl_connect_callback(self): - if self._ssl_connect_callback is not None: - callback = self._ssl_connect_callback - self._ssl_connect_callback = None - self._run_callback(callback) - if self._ssl_connect_future is not None: - future = self._ssl_connect_future - self._ssl_connect_future = None - future.set_result(self) - - def _verify_cert(self, peercert): - """Returns True if peercert is valid according to the configured - validation mode and hostname. - - The ssl handshake already tested the certificate for a valid - CA signature; the only thing that remains is to check - the hostname. - """ - if isinstance(self._ssl_options, dict): - verify_mode = self._ssl_options.get('cert_reqs', ssl.CERT_NONE) - elif isinstance(self._ssl_options, ssl.SSLContext): - verify_mode = self._ssl_options.verify_mode - assert verify_mode in (ssl.CERT_NONE, ssl.CERT_REQUIRED, ssl.CERT_OPTIONAL) - if verify_mode == ssl.CERT_NONE or self._server_hostname is None: - return True - cert = self.socket.getpeercert() - if cert is None and verify_mode == ssl.CERT_REQUIRED: - gen_log.warning("No SSL certificate given") - return False - try: - ssl_match_hostname(peercert, self._server_hostname) - except SSLCertificateError as e: - gen_log.warning("Invalid SSL certificate: %s" % e) - return False - else: - return True - - def _handle_read(self): - if self._ssl_accepting: - self._do_ssl_handshake() - return - super(SSLIOStream, self)._handle_read() - - def _handle_write(self): - if self._ssl_accepting: - self._do_ssl_handshake() - return - super(SSLIOStream, self)._handle_write() - - def connect(self, address, callback=None, server_hostname=None): - self._server_hostname = server_hostname - # Pass a dummy callback to super.connect(), which is slightly - # more efficient than letting it return a Future we ignore. - super(SSLIOStream, self).connect(address, callback=lambda: None) - return self.wait_for_handshake(callback) - - def _handle_connect(self): - # Call the superclass method to check for errors. - super(SSLIOStream, self)._handle_connect() - if self.closed(): - return - # When the connection is complete, wrap the socket for SSL - # traffic. Note that we do this by overriding _handle_connect - # instead of by passing a callback to super().connect because - # user callbacks are enqueued asynchronously on the IOLoop, - # but since _handle_events calls _handle_connect immediately - # followed by _handle_write we need this to be synchronous. - # - # The IOLoop will get confused if we swap out self.socket while the - # fd is registered, so remove it now and re-register after - # wrap_socket(). - self.io_loop.remove_handler(self.socket) - old_state = self._state - self._state = None - self.socket = ssl_wrap_socket(self.socket, self._ssl_options, - server_hostname=self._server_hostname, - do_handshake_on_connect=False) - self._add_io_state(old_state) - - def wait_for_handshake(self, callback=None): - """Wait for the initial SSL handshake to complete. - - If a ``callback`` is given, it will be called with no - arguments once the handshake is complete; otherwise this - method returns a `.Future` which will resolve to the - stream itself after the handshake is complete. - - Once the handshake is complete, information such as - the peer's certificate and NPN/ALPN selections may be - accessed on ``self.socket``. - - This method is intended for use on server-side streams - or after using `IOStream.start_tls`; it should not be used - with `IOStream.connect` (which already waits for the - handshake to complete). It may only be called once per stream. - - .. versionadded:: 4.2 - """ - if (self._ssl_connect_callback is not None or - self._ssl_connect_future is not None): - raise RuntimeError("Already waiting") - if callback is not None: - self._ssl_connect_callback = stack_context.wrap(callback) - future = None - else: - future = self._ssl_connect_future = TracebackFuture() - if not self._ssl_accepting: - self._run_ssl_connect_callback() - return future - - def write_to_fd(self, data): - try: - return self.socket.send(data) - except ssl.SSLError as e: - if e.args[0] == ssl.SSL_ERROR_WANT_WRITE: - # In Python 3.5+, SSLSocket.send raises a WANT_WRITE error if - # the socket is not writeable; we need to transform this into - # an EWOULDBLOCK socket.error or a zero return value, - # either of which will be recognized by the caller of this - # method. Prior to Python 3.5, an unwriteable socket would - # simply return 0 bytes written. - return 0 - raise - finally: - # Avoid keeping to data, which can be a memoryview. - # See https://github.com/tornadoweb/tornado/pull/2008 - del data - - def read_from_fd(self): - if self._ssl_accepting: - # If the handshake hasn't finished yet, there can't be anything - # to read (attempting to read may or may not raise an exception - # depending on the SSL version) - return None - try: - # SSLSocket objects have both a read() and recv() method, - # while regular sockets only have recv(). - # The recv() method blocks (at least in python 2.6) if it is - # called when there is nothing to read, so we have to use - # read() instead. - chunk = self.socket.read(self.read_chunk_size) - except ssl.SSLError as e: - # SSLError is a subclass of socket.error, so this except - # block must come first. - if e.args[0] == ssl.SSL_ERROR_WANT_READ: - return None - else: - raise - except socket.error as e: - if e.args[0] in _ERRNO_WOULDBLOCK: - return None - else: - raise - if not chunk: - self.close() - return None - return chunk - - def _is_connreset(self, e): - if isinstance(e, ssl.SSLError) and e.args[0] == ssl.SSL_ERROR_EOF: - return True - return super(SSLIOStream, self)._is_connreset(e) - - -class PipeIOStream(BaseIOStream): - """Pipe-based `IOStream` implementation. - - The constructor takes an integer file descriptor (such as one returned - by `os.pipe`) rather than an open file object. Pipes are generally - one-way, so a `PipeIOStream` can be used for reading or writing but not - both. - """ - def __init__(self, fd, *args, **kwargs): - self.fd = fd - _set_nonblocking(fd) - super(PipeIOStream, self).__init__(*args, **kwargs) - - def fileno(self): - return self.fd - - def close_fd(self): - os.close(self.fd) - - def write_to_fd(self, data): - try: - return os.write(self.fd, data) - finally: - # Avoid keeping to data, which can be a memoryview. - # See https://github.com/tornadoweb/tornado/pull/2008 - del data - - def read_from_fd(self): - try: - chunk = os.read(self.fd, self.read_chunk_size) - except (IOError, OSError) as e: - if errno_from_exception(e) in _ERRNO_WOULDBLOCK: - return None - elif errno_from_exception(e) == errno.EBADF: - # If the writing half of a pipe is closed, select will - # report it as readable but reads will fail with EBADF. - self.close(exc_info=True) - return None - else: - raise - if not chunk: - self.close() - return None - return chunk - - -def doctests(): - import doctest - return doctest.DocTestSuite() diff --git a/salt/ext/tornado/locale.py b/salt/ext/tornado/locale.py deleted file mode 100644 index 3ae56429c02..00000000000 --- a/salt/ext/tornado/locale.py +++ /dev/null @@ -1,522 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Translation methods for generating localized strings. - -To load a locale and generate a translated string:: - - user_locale = tornado.locale.get("es_LA") - print(user_locale.translate("Sign out")) - -`tornado.locale.get()` returns the closest matching locale, not necessarily the -specific locale you requested. You can support pluralization with -additional arguments to `~Locale.translate()`, e.g.:: - - people = [...] - message = user_locale.translate( - "%(list)s is online", "%(list)s are online", len(people)) - print(message % {"list": user_locale.list(people)}) - -The first string is chosen if ``len(people) == 1``, otherwise the second -string is chosen. - -Applications should call one of `load_translations` (which uses a simple -CSV format) or `load_gettext_translations` (which uses the ``.mo`` format -supported by `gettext` and related tools). If neither method is called, -the `Locale.translate` method will simply return the original string. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import codecs -import csv -import datetime -from io import BytesIO -import numbers -import os -import re - -from salt.ext.tornado import escape -from salt.ext.tornado.log import gen_log -from salt.ext.tornado.util import PY3 - -from salt.ext.tornado._locale_data import LOCALE_NAMES - -_default_locale = "en_US" -_translations = {} # type: dict -_supported_locales = frozenset([_default_locale]) -_use_gettext = False -CONTEXT_SEPARATOR = "\x04" - - -def get(*locale_codes): - """Returns the closest match for the given locale codes. - - We iterate over all given locale codes in order. If we have a tight - or a loose match for the code (e.g., "en" for "en_US"), we return - the locale. Otherwise we move to the next code in the list. - - By default we return ``en_US`` if no translations are found for any of - the specified locales. You can change the default locale with - `set_default_locale()`. - """ - return Locale.get_closest(*locale_codes) - - -def set_default_locale(code): - """Sets the default locale. - - The default locale is assumed to be the language used for all strings - in the system. The translations loaded from disk are mappings from - the default locale to the destination locale. Consequently, you don't - need to create a translation file for the default locale. - """ - global _default_locale - global _supported_locales - _default_locale = code - _supported_locales = frozenset(list(_translations.keys()) + [_default_locale]) - - -def load_translations(directory, encoding=None): - """Loads translations from CSV files in a directory. - - Translations are strings with optional Python-style named placeholders - (e.g., ``My name is %(name)s``) and their associated translations. - - The directory should have translation files of the form ``LOCALE.csv``, - e.g. ``es_GT.csv``. The CSV files should have two or three columns: string, - translation, and an optional plural indicator. Plural indicators should - be one of "plural" or "singular". A given string can have both singular - and plural forms. For example ``%(name)s liked this`` may have a - different verb conjugation depending on whether %(name)s is one - name or a list of names. There should be two rows in the CSV file for - that string, one with plural indicator "singular", and one "plural". - For strings with no verbs that would change on translation, simply - use "unknown" or the empty string (or don't include the column at all). - - The file is read using the `csv` module in the default "excel" dialect. - In this format there should not be spaces after the commas. - - If no ``encoding`` parameter is given, the encoding will be - detected automatically (among UTF-8 and UTF-16) if the file - contains a byte-order marker (BOM), defaulting to UTF-8 if no BOM - is present. - - Example translation ``es_LA.csv``:: - - "I love you","Te amo" - "%(name)s liked this","A %(name)s les gustó esto","plural" - "%(name)s liked this","A %(name)s le gustó esto","singular" - - .. versionchanged:: 4.3 - Added ``encoding`` parameter. Added support for BOM-based encoding - detection, UTF-16, and UTF-8-with-BOM. - """ - global _translations - global _supported_locales - _translations = {} - for path in os.listdir(directory): - if not path.endswith(".csv"): - continue - locale, extension = path.split(".") - if not re.match("[a-z]+(_[A-Z]+)?$", locale): - gen_log.error("Unrecognized locale %r (path: %s)", locale, - os.path.join(directory, path)) - continue - full_path = os.path.join(directory, path) - if encoding is None: - # Try to autodetect encoding based on the BOM. - with open(full_path, 'rb') as f: - data = f.read(len(codecs.BOM_UTF16_LE)) - if data in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE): - encoding = 'utf-16' - else: - # utf-8-sig is "utf-8 with optional BOM". It's discouraged - # in most cases but is common with CSV files because Excel - # cannot read utf-8 files without a BOM. - encoding = 'utf-8-sig' - if PY3: - # python 3: csv.reader requires a file open in text mode. - # Force utf8 to avoid dependence on $LANG environment variable. - f = open(full_path, "r", encoding=encoding) - else: - # python 2: csv can only handle byte strings (in ascii-compatible - # encodings), which we decode below. Transcode everything into - # utf8 before passing it to csv.reader. - f = BytesIO() - with codecs.open(full_path, "r", encoding=encoding) as infile: - f.write(escape.utf8(infile.read())) - f.seek(0) - _translations[locale] = {} - for i, row in enumerate(csv.reader(f)): - if not row or len(row) < 2: - continue - row = [escape.to_unicode(c).strip() for c in row] - english, translation = row[:2] - if len(row) > 2: - plural = row[2] or "unknown" - else: - plural = "unknown" - if plural not in ("plural", "singular", "unknown"): - gen_log.error("Unrecognized plural indicator %r in %s line %d", - plural, path, i + 1) - continue - _translations[locale].setdefault(plural, {})[english] = translation - f.close() - _supported_locales = frozenset(list(_translations.keys()) + [_default_locale]) - gen_log.debug("Supported locales: %s", sorted(_supported_locales)) - - -def load_gettext_translations(directory, domain): - """Loads translations from `gettext`'s locale tree - - Locale tree is similar to system's ``/usr/share/locale``, like:: - - {directory}/{lang}/LC_MESSAGES/{domain}.mo - - Three steps are required to have your app translated: - - 1. Generate POT translation file:: - - xgettext --language=Python --keyword=_:1,2 -d mydomain file1.py file2.html etc - - 2. Merge against existing POT file:: - - msgmerge old.po mydomain.po > new.po - - 3. Compile:: - - msgfmt mydomain.po -o {directory}/pt_BR/LC_MESSAGES/mydomain.mo - """ - import gettext - global _translations - global _supported_locales - global _use_gettext - _translations = {} - for lang in os.listdir(directory): - if lang.startswith('.'): - continue # skip .svn, etc - if os.path.isfile(os.path.join(directory, lang)): - continue - try: - os.stat(os.path.join(directory, lang, "LC_MESSAGES", domain + ".mo")) - _translations[lang] = gettext.translation(domain, directory, - languages=[lang]) - except Exception as e: - gen_log.error("Cannot load translation for '%s': %s", lang, str(e)) - continue - _supported_locales = frozenset(list(_translations.keys()) + [_default_locale]) - _use_gettext = True - gen_log.debug("Supported locales: %s", sorted(_supported_locales)) - - -def get_supported_locales(): - """Returns a list of all the supported locale codes.""" - return _supported_locales - - -class Locale(object): - """Object representing a locale. - - After calling one of `load_translations` or `load_gettext_translations`, - call `get` or `get_closest` to get a Locale object. - """ - @classmethod - def get_closest(cls, *locale_codes): - """Returns the closest match for the given locale code.""" - for code in locale_codes: - if not code: - continue - code = code.replace("-", "_") - parts = code.split("_") - if len(parts) > 2: - continue - elif len(parts) == 2: - code = parts[0].lower() + "_" + parts[1].upper() - if code in _supported_locales: - return cls.get(code) - if parts[0].lower() in _supported_locales: - return cls.get(parts[0].lower()) - return cls.get(_default_locale) - - @classmethod - def get(cls, code): - """Returns the Locale for the given locale code. - - If it is not supported, we raise an exception. - """ - if not hasattr(cls, "_cache"): - cls._cache = {} - if code not in cls._cache: - assert code in _supported_locales - translations = _translations.get(code, None) - if translations is None: - locale = CSVLocale(code, {}) - elif _use_gettext: - locale = GettextLocale(code, translations) - else: - locale = CSVLocale(code, translations) - cls._cache[code] = locale - return cls._cache[code] - - def __init__(self, code, translations): - self.code = code - self.name = LOCALE_NAMES.get(code, {}).get("name", u"Unknown") - self.rtl = False - for prefix in ["fa", "ar", "he"]: - if self.code.startswith(prefix): - self.rtl = True - break - self.translations = translations - - # Initialize strings for date formatting - _ = self.translate - self._months = [ - _("January"), _("February"), _("March"), _("April"), - _("May"), _("June"), _("July"), _("August"), - _("September"), _("October"), _("November"), _("December")] - self._weekdays = [ - _("Monday"), _("Tuesday"), _("Wednesday"), _("Thursday"), - _("Friday"), _("Saturday"), _("Sunday")] - - def translate(self, message, plural_message=None, count=None): - """Returns the translation for the given message for this locale. - - If ``plural_message`` is given, you must also provide - ``count``. We return ``plural_message`` when ``count != 1``, - and we return the singular form for the given message when - ``count == 1``. - """ - raise NotImplementedError() - - def pgettext(self, context, message, plural_message=None, count=None): - raise NotImplementedError() - - def format_date(self, date, gmt_offset=0, relative=True, shorter=False, - full_format=False): - """Formats the given date (which should be GMT). - - By default, we return a relative time (e.g., "2 minutes ago"). You - can return an absolute date string with ``relative=False``. - - You can force a full format date ("July 10, 1980") with - ``full_format=True``. - - This method is primarily intended for dates in the past. - For dates in the future, we fall back to full format. - """ - if isinstance(date, numbers.Real): - date = datetime.datetime.utcfromtimestamp(date) - now = datetime.datetime.utcnow() - if date > now: - if relative and (date - now).seconds < 60: - # Due to click skew, things are some things slightly - # in the future. Round timestamps in the immediate - # future down to now in relative mode. - date = now - else: - # Otherwise, future dates always use the full format. - full_format = True - local_date = date - datetime.timedelta(minutes=gmt_offset) - local_now = now - datetime.timedelta(minutes=gmt_offset) - local_yesterday = local_now - datetime.timedelta(hours=24) - difference = now - date - seconds = difference.seconds - days = difference.days - - _ = self.translate - format = None - if not full_format: - if relative and days == 0: - if seconds < 50: - return _("1 second ago", "%(seconds)d seconds ago", - seconds) % {"seconds": seconds} - - if seconds < 50 * 60: - minutes = round(seconds / 60.0) - return _("1 minute ago", "%(minutes)d minutes ago", - minutes) % {"minutes": minutes} - - hours = round(seconds / (60.0 * 60)) - return _("1 hour ago", "%(hours)d hours ago", - hours) % {"hours": hours} - - if days == 0: - format = _("%(time)s") - elif days == 1 and local_date.day == local_yesterday.day and \ - relative: - format = _("yesterday") if shorter else \ - _("yesterday at %(time)s") - elif days < 5: - format = _("%(weekday)s") if shorter else \ - _("%(weekday)s at %(time)s") - elif days < 334: # 11mo, since confusing for same month last year - format = _("%(month_name)s %(day)s") if shorter else \ - _("%(month_name)s %(day)s at %(time)s") - - if format is None: - format = _("%(month_name)s %(day)s, %(year)s") if shorter else \ - _("%(month_name)s %(day)s, %(year)s at %(time)s") - - tfhour_clock = self.code not in ("en", "en_US", "zh_CN") - if tfhour_clock: - str_time = "%d:%02d" % (local_date.hour, local_date.minute) - elif self.code == "zh_CN": - str_time = "%s%d:%02d" % ( - (u'\u4e0a\u5348', u'\u4e0b\u5348')[local_date.hour >= 12], - local_date.hour % 12 or 12, local_date.minute) - else: - str_time = "%d:%02d %s" % ( - local_date.hour % 12 or 12, local_date.minute, - ("am", "pm")[local_date.hour >= 12]) - - return format % { - "month_name": self._months[local_date.month - 1], - "weekday": self._weekdays[local_date.weekday()], - "day": str(local_date.day), - "year": str(local_date.year), - "time": str_time - } - - def format_day(self, date, gmt_offset=0, dow=True): - """Formats the given date as a day of week. - - Example: "Monday, January 22". You can remove the day of week with - ``dow=False``. - """ - local_date = date - datetime.timedelta(minutes=gmt_offset) - _ = self.translate - if dow: - return _("%(weekday)s, %(month_name)s %(day)s") % { - "month_name": self._months[local_date.month - 1], - "weekday": self._weekdays[local_date.weekday()], - "day": str(local_date.day), - } - else: - return _("%(month_name)s %(day)s") % { - "month_name": self._months[local_date.month - 1], - "day": str(local_date.day), - } - - def list(self, parts): - """Returns a comma-separated list for the given list of parts. - - The format is, e.g., "A, B and C", "A and B" or just "A" for lists - of size 1. - """ - _ = self.translate - if len(parts) == 0: - return "" - if len(parts) == 1: - return parts[0] - comma = u' \u0648 ' if self.code.startswith("fa") else u", " - return _("%(commas)s and %(last)s") % { - "commas": comma.join(parts[:-1]), - "last": parts[len(parts) - 1], - } - - def friendly_number(self, value): - """Returns a comma-separated number for the given integer.""" - if self.code not in ("en", "en_US"): - return str(value) - value = str(value) - parts = [] - while value: - parts.append(value[-3:]) - value = value[:-3] - return ",".join(reversed(parts)) - - -class CSVLocale(Locale): - """Locale implementation using tornado's CSV translation format.""" - def translate(self, message, plural_message=None, count=None): - if plural_message is not None: - assert count is not None - if count != 1: - message = plural_message - message_dict = self.translations.get("plural", {}) - else: - message_dict = self.translations.get("singular", {}) - else: - message_dict = self.translations.get("unknown", {}) - return message_dict.get(message, message) - - def pgettext(self, context, message, plural_message=None, count=None): - if self.translations: - gen_log.warning('pgettext is not supported by CSVLocale') - return self.translate(message, plural_message, count) - - -class GettextLocale(Locale): - """Locale implementation using the `gettext` module.""" - def __init__(self, code, translations): - try: - # python 2 - self.ngettext = translations.ungettext - self.gettext = translations.ugettext - except AttributeError: - # python 3 - self.ngettext = translations.ngettext - self.gettext = translations.gettext - # self.gettext must exist before __init__ is called, since it - # calls into self.translate - super(GettextLocale, self).__init__(code, translations) - - def translate(self, message, plural_message=None, count=None): - if plural_message is not None: - assert count is not None - return self.ngettext(message, plural_message, count) - else: - return self.gettext(message) - - def pgettext(self, context, message, plural_message=None, count=None): - """Allows to set context for translation, accepts plural forms. - - Usage example:: - - pgettext("law", "right") - pgettext("good", "right") - - Plural message example:: - - pgettext("organization", "club", "clubs", len(clubs)) - pgettext("stick", "club", "clubs", len(clubs)) - - To generate POT file with context, add following options to step 1 - of `load_gettext_translations` sequence:: - - xgettext [basic options] --keyword=pgettext:1c,2 --keyword=pgettext:1c,2,3 - - .. versionadded:: 4.2 - """ - if plural_message is not None: - assert count is not None - msgs_with_ctxt = ("%s%s%s" % (context, CONTEXT_SEPARATOR, message), - "%s%s%s" % (context, CONTEXT_SEPARATOR, plural_message), - count) - result = self.ngettext(*msgs_with_ctxt) - if CONTEXT_SEPARATOR in result: - # Translation not found - result = self.ngettext(message, plural_message, count) - return result - else: - msg_with_ctxt = "%s%s%s" % (context, CONTEXT_SEPARATOR, message) - result = self.gettext(msg_with_ctxt) - if CONTEXT_SEPARATOR in result: - # Translation not found - result = message - return result diff --git a/salt/ext/tornado/locks.py b/salt/ext/tornado/locks.py deleted file mode 100644 index dd1f7e1e314..00000000000 --- a/salt/ext/tornado/locks.py +++ /dev/null @@ -1,513 +0,0 @@ -# Copyright 2015 The Tornado Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import collections - -from salt.ext.tornado import gen, ioloop -from salt.ext.tornado.concurrent import Future - -__all__ = ['Condition', 'Event', 'Semaphore', 'BoundedSemaphore', 'Lock'] - - -class _TimeoutGarbageCollector(object): - """Base class for objects that periodically clean up timed-out waiters. - - Avoids memory leak in a common pattern like: - - while True: - yield condition.wait(short_timeout) - print('looping....') - """ - def __init__(self): - self._waiters = collections.deque() # Futures. - self._timeouts = 0 - - def _garbage_collect(self): - # Occasionally clear timed-out waiters. - self._timeouts += 1 - if self._timeouts > 100: - self._timeouts = 0 - self._waiters = collections.deque( - w for w in self._waiters if not w.done()) - - -class Condition(_TimeoutGarbageCollector): - """A condition allows one or more coroutines to wait until notified. - - Like a standard `threading.Condition`, but does not need an underlying lock - that is acquired and released. - - With a `Condition`, coroutines can wait to be notified by other coroutines: - - .. testcode:: - - from salt.ext.tornado import gen - from salt.ext.tornado.ioloop import IOLoop - from salt.ext.tornado.locks import Condition - - condition = Condition() - - @gen.coroutine - def waiter(): - print("I'll wait right here") - yield condition.wait() # Yield a Future. - print("I'm done waiting") - - @gen.coroutine - def notifier(): - print("About to notify") - condition.notify() - print("Done notifying") - - @gen.coroutine - def runner(): - # Yield two Futures; wait for waiter() and notifier() to finish. - yield [waiter(), notifier()] - - IOLoop.current().run_sync(runner) - - .. testoutput:: - - I'll wait right here - About to notify - Done notifying - I'm done waiting - - `wait` takes an optional ``timeout`` argument, which is either an absolute - timestamp:: - - io_loop = IOLoop.current() - - # Wait up to 1 second for a notification. - yield condition.wait(timeout=io_loop.time() + 1) - - ...or a `datetime.timedelta` for a timeout relative to the current time:: - - # Wait up to 1 second. - yield condition.wait(timeout=datetime.timedelta(seconds=1)) - - The method raises `tornado.gen.TimeoutError` if there's no notification - before the deadline. - """ - - def __init__(self): - super(Condition, self).__init__() - self.io_loop = ioloop.IOLoop.current() - - def __repr__(self): - result = '<%s' % (self.__class__.__name__, ) - if self._waiters: - result += ' waiters[%s]' % len(self._waiters) - return result + '>' - - def wait(self, timeout=None): - """Wait for `.notify`. - - Returns a `.Future` that resolves ``True`` if the condition is notified, - or ``False`` after a timeout. - """ - waiter = Future() - self._waiters.append(waiter) - if timeout: - def on_timeout(): - waiter.set_result(False) - self._garbage_collect() - io_loop = ioloop.IOLoop.current() - timeout_handle = io_loop.add_timeout(timeout, on_timeout) - waiter.add_done_callback( - lambda _: io_loop.remove_timeout(timeout_handle)) - return waiter - - def notify(self, n=1): - """Wake ``n`` waiters.""" - waiters = [] # Waiters we plan to run right now. - while n and self._waiters: - waiter = self._waiters.popleft() - if not waiter.done(): # Might have timed out. - n -= 1 - waiters.append(waiter) - - for waiter in waiters: - waiter.set_result(True) - - def notify_all(self): - """Wake all waiters.""" - self.notify(len(self._waiters)) - - -class Event(object): - """An event blocks coroutines until its internal flag is set to True. - - Similar to `threading.Event`. - - A coroutine can wait for an event to be set. Once it is set, calls to - ``yield event.wait()`` will not block unless the event has been cleared: - - .. testcode:: - - from salt.ext.tornado import gen - from salt.ext.tornado.ioloop import IOLoop - from salt.ext.tornado.locks import Event - - event = Event() - - @gen.coroutine - def waiter(): - print("Waiting for event") - yield event.wait() - print("Not waiting this time") - yield event.wait() - print("Done") - - @gen.coroutine - def setter(): - print("About to set the event") - event.set() - - @gen.coroutine - def runner(): - yield [waiter(), setter()] - - IOLoop.current().run_sync(runner) - - .. testoutput:: - - Waiting for event - About to set the event - Not waiting this time - Done - """ - def __init__(self): - self._future = Future() - - def __repr__(self): - return '<%s %s>' % ( - self.__class__.__name__, 'set' if self.is_set() else 'clear') - - def is_set(self): - """Return ``True`` if the internal flag is true.""" - return self._future.done() - - def set(self): - """Set the internal flag to ``True``. All waiters are awakened. - - Calling `.wait` once the flag is set will not block. - """ - if not self._future.done(): - self._future.set_result(None) - - def clear(self): - """Reset the internal flag to ``False``. - - Calls to `.wait` will block until `.set` is called. - """ - if self._future.done(): - self._future = Future() - - def wait(self, timeout=None): - """Block until the internal flag is true. - - Returns a Future, which raises `tornado.gen.TimeoutError` after a - timeout. - """ - if timeout is None: - return self._future - else: - return gen.with_timeout(timeout, self._future) - - -class _ReleasingContextManager(object): - """Releases a Lock or Semaphore at the end of a "with" statement. - - with (yield semaphore.acquire()): - pass - - # Now semaphore.release() has been called. - """ - def __init__(self, obj): - self._obj = obj - - def __enter__(self): - pass - - def __exit__(self, exc_type, exc_val, exc_tb): - self._obj.release() - - -class Semaphore(_TimeoutGarbageCollector): - """A lock that can be acquired a fixed number of times before blocking. - - A Semaphore manages a counter representing the number of `.release` calls - minus the number of `.acquire` calls, plus an initial value. The `.acquire` - method blocks if necessary until it can return without making the counter - negative. - - Semaphores limit access to a shared resource. To allow access for two - workers at a time: - - .. testsetup:: semaphore - - from collections import deque - - from salt.ext.tornado import gen - from salt.ext.tornado.ioloop import IOLoop - from salt.ext.tornado.concurrent import Future - - # Ensure reliable doctest output: resolve Futures one at a time. - futures_q = deque([Future() for _ in range(3)]) - - @gen.coroutine - def simulator(futures): - for f in futures: - yield gen.moment - f.set_result(None) - - IOLoop.current().add_callback(simulator, list(futures_q)) - - def use_some_resource(): - return futures_q.popleft() - - .. testcode:: semaphore - - from salt.ext.tornado import gen - from salt.ext.tornado.ioloop import IOLoop - from salt.ext.tornado.locks import Semaphore - - sem = Semaphore(2) - - @gen.coroutine - def worker(worker_id): - yield sem.acquire() - try: - print("Worker %d is working" % worker_id) - yield use_some_resource() - finally: - print("Worker %d is done" % worker_id) - sem.release() - - @gen.coroutine - def runner(): - # Join all workers. - yield [worker(i) for i in range(3)] - - IOLoop.current().run_sync(runner) - - .. testoutput:: semaphore - - Worker 0 is working - Worker 1 is working - Worker 0 is done - Worker 2 is working - Worker 1 is done - Worker 2 is done - - Workers 0 and 1 are allowed to run concurrently, but worker 2 waits until - the semaphore has been released once, by worker 0. - - `.acquire` is a context manager, so ``worker`` could be written as:: - - @gen.coroutine - def worker(worker_id): - with (yield sem.acquire()): - print("Worker %d is working" % worker_id) - yield use_some_resource() - - # Now the semaphore has been released. - print("Worker %d is done" % worker_id) - - In Python 3.5, the semaphore itself can be used as an async context - manager:: - - async def worker(worker_id): - async with sem: - print("Worker %d is working" % worker_id) - await use_some_resource() - - # Now the semaphore has been released. - print("Worker %d is done" % worker_id) - - .. versionchanged:: 4.3 - Added ``async with`` support in Python 3.5. - """ - def __init__(self, value=1): - super(Semaphore, self).__init__() - if value < 0: - raise ValueError('semaphore initial value must be >= 0') - - self._value = value - - def __repr__(self): - res = super(Semaphore, self).__repr__() - extra = 'locked' if self._value == 0 else 'unlocked,value:{0}'.format( - self._value) - if self._waiters: - extra = '{0},waiters:{1}'.format(extra, len(self._waiters)) - return '<{0} [{1}]>'.format(res[1:-1], extra) - - def release(self): - """Increment the counter and wake one waiter.""" - self._value += 1 - while self._waiters: - waiter = self._waiters.popleft() - if not waiter.done(): - self._value -= 1 - - # If the waiter is a coroutine paused at - # - # with (yield semaphore.acquire()): - # - # then the context manager's __exit__ calls release() at the end - # of the "with" block. - waiter.set_result(_ReleasingContextManager(self)) - break - - def acquire(self, timeout=None): - """Decrement the counter. Returns a Future. - - Block if the counter is zero and wait for a `.release`. The Future - raises `.TimeoutError` after the deadline. - """ - waiter = Future() - if self._value > 0: - self._value -= 1 - waiter.set_result(_ReleasingContextManager(self)) - else: - self._waiters.append(waiter) - if timeout: - def on_timeout(): - waiter.set_exception(gen.TimeoutError()) - self._garbage_collect() - io_loop = ioloop.IOLoop.current() - timeout_handle = io_loop.add_timeout(timeout, on_timeout) - waiter.add_done_callback( - lambda _: io_loop.remove_timeout(timeout_handle)) - return waiter - - def __enter__(self): - raise RuntimeError( - "Use Semaphore like 'with (yield semaphore.acquire())', not like" - " 'with semaphore'") - - __exit__ = __enter__ - - @gen.coroutine - def __aenter__(self): - yield self.acquire() - - @gen.coroutine - def __aexit__(self, typ, value, tb): - self.release() - - -class BoundedSemaphore(Semaphore): - """A semaphore that prevents release() being called too many times. - - If `.release` would increment the semaphore's value past the initial - value, it raises `ValueError`. Semaphores are mostly used to guard - resources with limited capacity, so a semaphore released too many times - is a sign of a bug. - """ - def __init__(self, value=1): - super(BoundedSemaphore, self).__init__(value=value) - self._initial_value = value - - def release(self): - """Increment the counter and wake one waiter.""" - if self._value >= self._initial_value: - raise ValueError("Semaphore released too many times") - super(BoundedSemaphore, self).release() - - -class Lock(object): - """A lock for coroutines. - - A Lock begins unlocked, and `acquire` locks it immediately. While it is - locked, a coroutine that yields `acquire` waits until another coroutine - calls `release`. - - Releasing an unlocked lock raises `RuntimeError`. - - `acquire` supports the context manager protocol in all Python versions: - - >>> from salt.ext.tornado import gen, locks - >>> lock = locks.Lock() - >>> - >>> @gen.coroutine - ... def f(): - ... with (yield lock.acquire()): - ... # Do something holding the lock. - ... pass - ... - ... # Now the lock is released. - - In Python 3.5, `Lock` also supports the async context manager - protocol. Note that in this case there is no `acquire`, because - ``async with`` includes both the ``yield`` and the ``acquire`` - (just as it does with `threading.Lock`): - - >>> async def f(): # doctest: +SKIP - ... async with lock: - ... # Do something holding the lock. - ... pass - ... - ... # Now the lock is released. - - .. versionchanged:: 4.3 - Added ``async with`` support in Python 3.5. - - """ - def __init__(self): - self._block = BoundedSemaphore(value=1) - - def __repr__(self): - return "<%s _block=%s>" % ( - self.__class__.__name__, - self._block) - - def acquire(self, timeout=None): - """Attempt to lock. Returns a Future. - - Returns a Future, which raises `tornado.gen.TimeoutError` after a - timeout. - """ - return self._block.acquire(timeout) - - def release(self): - """Unlock. - - The first coroutine in line waiting for `acquire` gets the lock. - - If not locked, raise a `RuntimeError`. - """ - try: - self._block.release() - except ValueError: - raise RuntimeError('release unlocked lock') - - def __enter__(self): - raise RuntimeError( - "Use Lock like 'with (yield lock)', not like 'with lock'") - - __exit__ = __enter__ - - @gen.coroutine - def __aenter__(self): - yield self.acquire() - - @gen.coroutine - def __aexit__(self, typ, value, tb): - self.release() diff --git a/salt/ext/tornado/log.py b/salt/ext/tornado/log.py deleted file mode 100644 index 7f9df691c0e..00000000000 --- a/salt/ext/tornado/log.py +++ /dev/null @@ -1,291 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Logging support for Tornado. - -Tornado uses three logger streams: - -* ``tornado.access``: Per-request logging for Tornado's HTTP servers (and - potentially other servers in the future) -* ``tornado.application``: Logging of errors from application code (i.e. - uncaught exceptions from callbacks) -* ``tornado.general``: General-purpose logging, including any errors - or warnings from Tornado itself. - -These streams may be configured independently using the standard library's -`logging` module. For example, you may wish to send ``tornado.access`` logs -to a separate file for analysis. -""" -# pylint: skip-file -from __future__ import absolute_import, division, print_function - -import logging -import logging.handlers -import sys - -from salt.ext.tornado.escape import _unicode -from salt.ext.tornado.util import unicode_type, basestring_type - -try: - import colorama -except ImportError: - colorama = None - -try: - import curses # type: ignore -except ImportError: - curses = None - -# Logger objects for internal tornado use -access_log = logging.getLogger("tornado.access") -app_log = logging.getLogger("tornado.application") -gen_log = logging.getLogger("tornado.general") - - -def _stderr_supports_color(): - try: - if hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): - if curses: - curses.setupterm() - if curses.tigetnum("colors") > 0: - return True - elif colorama: - if sys.stderr is getattr(colorama.initialise, 'wrapped_stderr', - object()): - return True - except Exception: - # Very broad exception handling because it's always better to - # fall back to non-colored logs than to break at startup. - pass - return False - - -def _safe_unicode(s): - try: - return _unicode(s) - except UnicodeDecodeError: - return repr(s) - - -class LogFormatter(logging.Formatter): - """Log formatter used in Tornado. - - Key features of this formatter are: - - * Color support when logging to a terminal that supports it. - * Timestamps on every log line. - * Robust against str/bytes encoding problems. - - This formatter is enabled automatically by - `tornado.options.parse_command_line` or `tornado.options.parse_config_file` - (unless ``--logging=none`` is used). - - Color support on Windows versions that do not support ANSI color codes is - enabled by use of the colorama__ library. Applications that wish to use - this must first initialize colorama with a call to ``colorama.init``. - See the colorama documentation for details. - - __ https://pypi.python.org/pypi/colorama - - .. versionchanged:: 4.5 - Added support for ``colorama``. Changed the constructor - signature to be compatible with `logging.config.dictConfig`. - """ - DEFAULT_FORMAT = '%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s' - DEFAULT_DATE_FORMAT = '%y%m%d %H:%M:%S' - DEFAULT_COLORS = { - logging.DEBUG: 4, # Blue - logging.INFO: 2, # Green - logging.WARNING: 3, # Yellow - logging.ERROR: 1, # Red - } - - def __init__(self, fmt=DEFAULT_FORMAT, datefmt=DEFAULT_DATE_FORMAT, - style='%', color=True, colors=DEFAULT_COLORS): - r""" - :arg bool color: Enables color support. - :arg string fmt: Log message format. - It will be applied to the attributes dict of log records. The - text between ``%(color)s`` and ``%(end_color)s`` will be colored - depending on the level if color support is on. - :arg dict colors: color mappings from logging level to terminal color - code - :arg string datefmt: Datetime format. - Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. - - .. versionchanged:: 3.2 - - Added ``fmt`` and ``datefmt`` arguments. - """ - logging.Formatter.__init__(self, datefmt=datefmt) - self._fmt = fmt - - self._colors = {} - if color and _stderr_supports_color(): - if curses is not None: - # The curses module has some str/bytes confusion in - # python3. Until version 3.2.3, most methods return - # bytes, but only accept strings. In addition, we want to - # output these strings with the logging module, which - # works with unicode strings. The explicit calls to - # unicode() below are harmless in python2 but will do the - # right conversion in python 3. - fg_color = (curses.tigetstr("setaf") or - curses.tigetstr("setf") or "") - if (3, 0) < sys.version_info < (3, 2, 3): - fg_color = unicode_type(fg_color, "ascii") - - for levelno, code in colors.items(): - self._colors[levelno] = unicode_type(curses.tparm(fg_color, code), "ascii") - self._normal = unicode_type(curses.tigetstr("sgr0"), "ascii") - else: - # If curses is not present (currently we'll only get here for - # colorama on windows), assume hard-coded ANSI color codes. - for levelno, code in colors.items(): - self._colors[levelno] = '\033[2;3%dm' % code - self._normal = '\033[0m' - else: - self._normal = '' - - def format(self, record): - try: - message = record.getMessage() - assert isinstance(message, basestring_type) # guaranteed by logging - # Encoding notes: The logging module prefers to work with character - # strings, but only enforces that log messages are instances of - # basestring. In python 2, non-ascii bytestrings will make - # their way through the logging framework until they blow up with - # an unhelpful decoding error (with this formatter it happens - # when we attach the prefix, but there are other opportunities for - # exceptions further along in the framework). - # - # If a byte string makes it this far, convert it to unicode to - # ensure it will make it out to the logs. Use repr() as a fallback - # to ensure that all byte strings can be converted successfully, - # but don't do it by default so we don't add extra quotes to ascii - # bytestrings. This is a bit of a hacky place to do this, but - # it's worth it since the encoding errors that would otherwise - # result are so useless (and tornado is fond of using utf8-encoded - # byte strings whereever possible). - record.message = _safe_unicode(message) - except Exception as e: - record.message = "Bad message (%r): %r" % (e, record.__dict__) - - record.asctime = self.formatTime(record, self.datefmt) - - if record.levelno in self._colors: - record.color = self._colors[record.levelno] - record.end_color = self._normal - else: - record.color = record.end_color = '' - - formatted = self._fmt % record.__dict__ - - if record.exc_info: - if not record.exc_text: - record.exc_text = self.formatException(record.exc_info) - if record.exc_text: - # exc_text contains multiple lines. We need to _safe_unicode - # each line separately so that non-utf8 bytes don't cause - # all the newlines to turn into '\n'. - lines = [formatted.rstrip()] - lines.extend(_safe_unicode(ln) for ln in record.exc_text.split('\n')) - formatted = '\n'.join(lines) - return formatted.replace("\n", "\n ") - - -def enable_pretty_logging(options=None, logger=None): - """Turns on formatted logging output as configured. - - This is called automatically by `tornado.options.parse_command_line` - and `tornado.options.parse_config_file`. - """ - if options is None: - import salt.ext.tornado.options - options = salt.ext.tornado.options.options - if options.logging is None or options.logging.lower() == 'none': - return - if logger is None: - logger = logging.getLogger() - logger.setLevel(getattr(logging, options.logging.upper())) - if options.log_file_prefix: - rotate_mode = options.log_rotate_mode - if rotate_mode == 'size': - channel = logging.handlers.RotatingFileHandler( - filename=options.log_file_prefix, - maxBytes=options.log_file_max_size, - backupCount=options.log_file_num_backups) - elif rotate_mode == 'time': - channel = logging.handlers.TimedRotatingFileHandler( - filename=options.log_file_prefix, - when=options.log_rotate_when, - interval=options.log_rotate_interval, - backupCount=options.log_file_num_backups) - else: - error_message = 'The value of log_rotate_mode option should be ' +\ - '"size" or "time", not "%s".' % rotate_mode - raise ValueError(error_message) - channel.setFormatter(LogFormatter(color=False)) - logger.addHandler(channel) - - if (options.log_to_stderr or - (options.log_to_stderr is None and not logger.handlers)): - # Set up color if we are in a tty and curses is installed - channel = logging.StreamHandler() - channel.setFormatter(LogFormatter()) - logger.addHandler(channel) - - -def define_logging_options(options=None): - """Add logging-related flags to ``options``. - - These options are present automatically on the default options instance; - this method is only necessary if you have created your own `.OptionParser`. - - .. versionadded:: 4.2 - This function existed in prior versions but was broken and undocumented until 4.2. - """ - if options is None: - # late import to prevent cycle - import salt.ext.tornado.options - options = salt.ext.tornado.options.options - options.define("logging", default="info", - help=("Set the Python log level. If 'none', tornado won't touch the " - "logging configuration."), - metavar="debug|info|warning|error|none") - options.define("log_to_stderr", type=bool, default=None, - help=("Send log output to stderr (colorized if possible). " - "By default use stderr if --log_file_prefix is not set and " - "no other logging is configured.")) - options.define("log_file_prefix", type=str, default=None, metavar="PATH", - help=("Path prefix for log files. " - "Note that if you are running multiple tornado processes, " - "log_file_prefix must be different for each of them (e.g. " - "include the port number)")) - options.define("log_file_max_size", type=int, default=100 * 1000 * 1000, - help="max size of log files before rollover") - options.define("log_file_num_backups", type=int, default=10, - help="number of log files to keep") - - options.define("log_rotate_when", type=str, default='midnight', - help=("specify the type of TimedRotatingFileHandler interval " - "other options:('S', 'M', 'H', 'D', 'W0'-'W6')")) - options.define("log_rotate_interval", type=int, default=1, - help="The interval value of timed rotating") - - options.define("log_rotate_mode", type=str, default='size', - help="The mode of rotating files(time or size)") - - options.add_parse_callback(lambda: enable_pretty_logging(options)) diff --git a/salt/ext/tornado/netutil.py b/salt/ext/tornado/netutil.py deleted file mode 100644 index f86b430674b..00000000000 --- a/salt/ext/tornado/netutil.py +++ /dev/null @@ -1,530 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2011 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Miscellaneous network utility code.""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import errno -import os -import sys -import socket -import stat - -from salt.ext.tornado.concurrent import dummy_executor, run_on_executor -from salt.ext.tornado.ioloop import IOLoop -from salt.ext.tornado.platform.auto import set_close_exec -from salt.ext.tornado.util import PY3, Configurable, errno_from_exception - -try: - import ssl -except ImportError: - # ssl is not available on Google App Engine - ssl = None - -try: - import certifi -except ImportError: - # certifi is optional as long as we have ssl.create_default_context. - if ssl is None or hasattr(ssl, 'create_default_context'): - certifi = None - else: - raise - -if PY3: - xrange = range - -if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'): # python 3.2+ - ssl_match_hostname = ssl.match_hostname - SSLCertificateError = ssl.CertificateError -elif ssl is None: - ssl_match_hostname = SSLCertificateError = None # type: ignore -else: - import backports.ssl_match_hostname - ssl_match_hostname = backports.ssl_match_hostname.match_hostname - SSLCertificateError = backports.ssl_match_hostname.CertificateError # type: ignore - -if hasattr(ssl, 'SSLContext'): - if hasattr(ssl, 'create_default_context'): - # Python 2.7.9+, 3.4+ - # Note that the naming of ssl.Purpose is confusing; the purpose - # of a context is to authentiate the opposite side of the connection. - _client_ssl_defaults = ssl.create_default_context( - ssl.Purpose.SERVER_AUTH) - _server_ssl_defaults = ssl.create_default_context( - ssl.Purpose.CLIENT_AUTH) - else: - # Python 3.2-3.3 - _client_ssl_defaults = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - _client_ssl_defaults.verify_mode = ssl.CERT_REQUIRED - _client_ssl_defaults.load_verify_locations(certifi.where()) - _server_ssl_defaults = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - if hasattr(ssl, 'OP_NO_COMPRESSION'): - # Disable TLS compression to avoid CRIME and related attacks. - # This constant wasn't added until python 3.3. - _client_ssl_defaults.options |= ssl.OP_NO_COMPRESSION - _server_ssl_defaults.options |= ssl.OP_NO_COMPRESSION - -elif ssl: - # Python 2.6-2.7.8 - _client_ssl_defaults = dict(cert_reqs=ssl.CERT_REQUIRED, - ca_certs=certifi.where()) - _server_ssl_defaults = {} -else: - # Google App Engine - _client_ssl_defaults = dict(cert_reqs=None, - ca_certs=None) - _server_ssl_defaults = {} - -# ThreadedResolver runs getaddrinfo on a thread. If the hostname is unicode, -# getaddrinfo attempts to import encodings.idna. If this is done at -# module-import time, the import lock is already held by the main thread, -# leading to deadlock. Avoid it by caching the idna encoder on the main -# thread now. -u'foo'.encode('idna') - -# For undiagnosed reasons, 'latin1' codec may also need to be preloaded. -u'foo'.encode('latin1') - -# These errnos indicate that a non-blocking operation must be retried -# at a later time. On most platforms they're the same value, but on -# some they differ. -_ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN) - -if hasattr(errno, "WSAEWOULDBLOCK"): - _ERRNO_WOULDBLOCK += (errno.WSAEWOULDBLOCK,) # type: ignore - -# Default backlog used when calling sock.listen() -_DEFAULT_BACKLOG = 128 - - -def bind_sockets(port, address=None, family=socket.AF_UNSPEC, - backlog=_DEFAULT_BACKLOG, flags=None, reuse_port=False): - """Creates listening sockets bound to the given port and address. - - Returns a list of socket objects (multiple sockets are returned if - the given address maps to multiple IP addresses, which is most common - for mixed IPv4 and IPv6 use). - - Address may be either an IP address or hostname. If it's a hostname, - the server will listen on all IP addresses associated with the - name. Address may be an empty string or None to listen on all - available interfaces. Family may be set to either `socket.AF_INET` - or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise - both will be used if available. - - The ``backlog`` argument has the same meaning as for - `socket.listen() `. - - ``flags`` is a bitmask of AI_* flags to `~socket.getaddrinfo`, like - ``socket.AI_PASSIVE | socket.AI_NUMERICHOST``. - - ``reuse_port`` option sets ``SO_REUSEPORT`` option for every socket - in the list. If your platform doesn't support this option ValueError will - be raised. - """ - if reuse_port and not hasattr(socket, "SO_REUSEPORT"): - raise ValueError("the platform doesn't support SO_REUSEPORT") - - sockets = [] - if address == "": - address = None - if not socket.has_ipv6 and family == socket.AF_UNSPEC: - # Python can be compiled with --disable-ipv6, which causes - # operations on AF_INET6 sockets to fail, but does not - # automatically exclude those results from getaddrinfo - # results. - # http://bugs.python.org/issue16208 - family = socket.AF_INET - if flags is None: - flags = socket.AI_PASSIVE - bound_port = None - for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM, - 0, flags)): - af, socktype, proto, canonname, sockaddr = res - if (sys.platform == 'darwin' and address == 'localhost' and - af == socket.AF_INET6 and sockaddr[3] != 0): - # Mac OS X includes a link-local address fe80::1%lo0 in the - # getaddrinfo results for 'localhost'. However, the firewall - # doesn't understand that this is a local address and will - # prompt for access (often repeatedly, due to an apparent - # bug in its ability to remember granting access to an - # application). Skip these addresses. - continue - try: - sock = socket.socket(af, socktype, proto) - except socket.error as e: - if errno_from_exception(e) == errno.EAFNOSUPPORT: - continue - raise - set_close_exec(sock.fileno()) - if os.name != 'nt': - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if reuse_port: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - if af == socket.AF_INET6: - # On linux, ipv6 sockets accept ipv4 too by default, - # but this makes it impossible to bind to both - # 0.0.0.0 in ipv4 and :: in ipv6. On other systems, - # separate sockets *must* be used to listen for both ipv4 - # and ipv6. For consistency, always disable ipv4 on our - # ipv6 sockets and use a separate ipv4 socket when needed. - # - # Python 2.x on windows doesn't have IPPROTO_IPV6. - if hasattr(socket, "IPPROTO_IPV6"): - sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) - - # automatic port allocation with port=None - # should bind on the same port on IPv4 and IPv6 - host, requested_port = sockaddr[:2] - if requested_port == 0 and bound_port is not None: - sockaddr = tuple([host, bound_port] + list(sockaddr[2:])) - - sock.setblocking(0) - sock.bind(sockaddr) - bound_port = sock.getsockname()[1] - sock.listen(backlog) - sockets.append(sock) - return sockets - - -if hasattr(socket, 'AF_UNIX'): - def bind_unix_socket(file, mode=0o600, backlog=_DEFAULT_BACKLOG): - """Creates a listening unix socket. - - If a socket with the given name already exists, it will be deleted. - If any other file with that name exists, an exception will be - raised. - - Returns a socket object (not a list of socket objects like - `bind_sockets`) - """ - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - set_close_exec(sock.fileno()) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.setblocking(0) - try: - st = os.stat(file) - except OSError as err: - if errno_from_exception(err) != errno.ENOENT: - raise - else: - if stat.S_ISSOCK(st.st_mode): - os.remove(file) - else: - raise ValueError("File %s exists and is not a socket", file) - sock.bind(file) - os.chmod(file, mode) - sock.listen(backlog) - return sock - - -def add_accept_handler(sock, callback, io_loop=None): - """Adds an `.IOLoop` event handler to accept new connections on ``sock``. - - When a connection is accepted, ``callback(connection, address)`` will - be run (``connection`` is a socket object, and ``address`` is the - address of the other end of the connection). Note that this signature - is different from the ``callback(fd, events)`` signature used for - `.IOLoop` handlers. - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - """ - if io_loop is None: - io_loop = IOLoop.current() - - def accept_handler(fd, events): - # More connections may come in while we're handling callbacks; - # to prevent starvation of other tasks we must limit the number - # of connections we accept at a time. Ideally we would accept - # up to the number of connections that were waiting when we - # entered this method, but this information is not available - # (and rearranging this method to call accept() as many times - # as possible before running any callbacks would have adverse - # effects on load balancing in multiprocess configurations). - # Instead, we use the (default) listen backlog as a rough - # heuristic for the number of connections we can reasonably - # accept at once. - for i in xrange(_DEFAULT_BACKLOG): - try: - connection, address = sock.accept() - except socket.error as e: - # _ERRNO_WOULDBLOCK indicate we have accepted every - # connection that is available. - if errno_from_exception(e) in _ERRNO_WOULDBLOCK: - return - # ECONNABORTED indicates that there was a connection - # but it was closed while still in the accept queue. - # (observed on FreeBSD). - if errno_from_exception(e) == errno.ECONNABORTED: - continue - raise - set_close_exec(connection.fileno()) - callback(connection, address) - io_loop.add_handler(sock, accept_handler, IOLoop.READ) - - -def is_valid_ip(ip): - """Returns true if the given string is a well-formed IP address. - - Supports IPv4 and IPv6. - """ - if not ip or '\x00' in ip: - # getaddrinfo resolves empty strings to localhost, and truncates - # on zero bytes. - return False - try: - res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC, - socket.SOCK_STREAM, - 0, socket.AI_NUMERICHOST) - return bool(res) - except socket.gaierror as e: - if e.args[0] == socket.EAI_NONAME: - return False - raise - return True - - -class Resolver(Configurable): - """Configurable asynchronous DNS resolver interface. - - By default, a blocking implementation is used (which simply calls - `socket.getaddrinfo`). An alternative implementation can be - chosen with the `Resolver.configure <.Configurable.configure>` - class method:: - - Resolver.configure('tornado.netutil.ThreadedResolver') - - The implementations of this interface included with Tornado are - - * `tornado.netutil.BlockingResolver` - * `tornado.netutil.ThreadedResolver` - * `tornado.netutil.OverrideResolver` - * `tornado.platform.twisted.TwistedResolver` - * `tornado.platform.caresresolver.CaresResolver` - """ - @classmethod - def configurable_base(cls): - return Resolver - - @classmethod - def configurable_default(cls): - return BlockingResolver - - def resolve(self, host, port, family=socket.AF_UNSPEC, callback=None): - """Resolves an address. - - The ``host`` argument is a string which may be a hostname or a - literal IP address. - - Returns a `.Future` whose result is a list of (family, - address) pairs, where address is a tuple suitable to pass to - `socket.connect ` (i.e. a ``(host, - port)`` pair for IPv4; additional fields may be present for - IPv6). If a ``callback`` is passed, it will be run with the - result as an argument when it is complete. - - :raises IOError: if the address cannot be resolved. - - .. versionchanged:: 4.4 - Standardized all implementations to raise `IOError`. - """ - raise NotImplementedError() - - def close(self): - """Closes the `Resolver`, freeing any resources used. - - .. versionadded:: 3.1 - - """ - pass - - -class ExecutorResolver(Resolver): - """Resolver implementation using a `concurrent.futures.Executor`. - - Use this instead of `ThreadedResolver` when you require additional - control over the executor being used. - - The executor will be shut down when the resolver is closed unless - ``close_resolver=False``; use this if you want to reuse the same - executor elsewhere. - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - """ - def initialize(self, io_loop=None, executor=None, close_executor=True): - self.io_loop = io_loop or IOLoop.current() - if executor is not None: - self.executor = executor - self.close_executor = close_executor - else: - self.executor = dummy_executor - self.close_executor = False - - def close(self): - if self.close_executor: - self.executor.shutdown() - self.executor = None - - @run_on_executor - def resolve(self, host, port, family=socket.AF_UNSPEC): - # On Solaris, getaddrinfo fails if the given port is not found - # in /etc/services and no socket type is given, so we must pass - # one here. The socket type used here doesn't seem to actually - # matter (we discard the one we get back in the results), - # so the addresses we return should still be usable with SOCK_DGRAM. - addrinfo = socket.getaddrinfo(host, port, family, socket.SOCK_STREAM) - results = [] - for family, socktype, proto, canonname, address in addrinfo: - results.append((family, address)) - return results - - -class BlockingResolver(ExecutorResolver): - """Default `Resolver` implementation, using `socket.getaddrinfo`. - - The `.IOLoop` will be blocked during the resolution, although the - callback will not be run until the next `.IOLoop` iteration. - """ - def initialize(self, io_loop=None): - super(BlockingResolver, self).initialize(io_loop=io_loop) - - -class ThreadedResolver(ExecutorResolver): - """Multithreaded non-blocking `Resolver` implementation. - - Requires the `concurrent.futures` package to be installed - (available in the standard library since Python 3.2, - installable with ``pip install futures`` in older versions). - - The thread pool size can be configured with:: - - Resolver.configure('tornado.netutil.ThreadedResolver', - num_threads=10) - - .. versionchanged:: 3.1 - All ``ThreadedResolvers`` share a single thread pool, whose - size is set by the first one to be created. - """ - _threadpool = None # type: ignore - _threadpool_pid = None # type: int - - def initialize(self, io_loop=None, num_threads=10): - threadpool = ThreadedResolver._create_threadpool(num_threads) - super(ThreadedResolver, self).initialize( - io_loop=io_loop, executor=threadpool, close_executor=False) - - @classmethod - def _create_threadpool(cls, num_threads): - pid = os.getpid() - if cls._threadpool_pid != pid: - # Threads cannot survive after a fork, so if our pid isn't what it - # was when we created the pool then delete it. - cls._threadpool = None - if cls._threadpool is None: - from concurrent.futures import ThreadPoolExecutor - cls._threadpool = ThreadPoolExecutor(num_threads) - cls._threadpool_pid = pid - return cls._threadpool - - -class OverrideResolver(Resolver): - """Wraps a resolver with a mapping of overrides. - - This can be used to make local DNS changes (e.g. for testing) - without modifying system-wide settings. - - The mapping can contain either host strings or host-port pairs. - """ - def initialize(self, resolver, mapping): - self.resolver = resolver - self.mapping = mapping - - def close(self): - self.resolver.close() - - def resolve(self, host, port, *args, **kwargs): - if (host, port) in self.mapping: - host, port = self.mapping[(host, port)] - elif host in self.mapping: - host = self.mapping[host] - return self.resolver.resolve(host, port, *args, **kwargs) - - -# These are the keyword arguments to ssl.wrap_socket that must be translated -# to their SSLContext equivalents (the other arguments are still passed -# to SSLContext.wrap_socket). -_SSL_CONTEXT_KEYWORDS = frozenset(['ssl_version', 'certfile', 'keyfile', - 'cert_reqs', 'ca_certs', 'ciphers']) - - -def ssl_options_to_context(ssl_options): - """Try to convert an ``ssl_options`` dictionary to an - `~ssl.SSLContext` object. - - The ``ssl_options`` dictionary contains keywords to be passed to - `ssl.wrap_socket`. In Python 2.7.9+, `ssl.SSLContext` objects can - be used instead. This function converts the dict form to its - `~ssl.SSLContext` equivalent, and may be used when a component which - accepts both forms needs to upgrade to the `~ssl.SSLContext` version - to use features like SNI or NPN. - """ - if isinstance(ssl_options, dict): - assert all(k in _SSL_CONTEXT_KEYWORDS for k in ssl_options), ssl_options - if (not hasattr(ssl, 'SSLContext') or - isinstance(ssl_options, ssl.SSLContext)): - return ssl_options - context = ssl.SSLContext( - ssl_options.get('ssl_version', ssl.PROTOCOL_SSLv23)) - if 'certfile' in ssl_options: - context.load_cert_chain(ssl_options['certfile'], ssl_options.get('keyfile', None)) - if 'cert_reqs' in ssl_options: - context.verify_mode = ssl_options['cert_reqs'] - if 'ca_certs' in ssl_options: - context.load_verify_locations(ssl_options['ca_certs']) - if 'ciphers' in ssl_options: - context.set_ciphers(ssl_options['ciphers']) - if hasattr(ssl, 'OP_NO_COMPRESSION'): - # Disable TLS compression to avoid CRIME and related attacks. - # This constant wasn't added until python 3.3. - context.options |= ssl.OP_NO_COMPRESSION - return context - - -def ssl_wrap_socket(socket, ssl_options, server_hostname=None, **kwargs): - """Returns an ``ssl.SSLSocket`` wrapping the given socket. - - ``ssl_options`` may be either an `ssl.SSLContext` object or a - dictionary (as accepted by `ssl_options_to_context`). Additional - keyword arguments are passed to ``wrap_socket`` (either the - `~ssl.SSLContext` method or the `ssl` module function as - appropriate). - """ - context = ssl_options_to_context(ssl_options) - if hasattr(ssl, 'SSLContext') and isinstance(context, ssl.SSLContext): - if server_hostname is not None and getattr(ssl, 'HAS_SNI'): - # Python doesn't have server-side SNI support so we can't - # really unittest this, but it can be manually tested with - # python3.2 -m tornado.httpclient https://sni.velox.ch - return context.wrap_socket(socket, server_hostname=server_hostname, - **kwargs) - else: - return context.wrap_socket(socket, **kwargs) - else: - return ssl.wrap_socket(socket, **dict(context, **kwargs)) # type: ignore diff --git a/salt/ext/tornado/options.py b/salt/ext/tornado/options.py deleted file mode 100644 index d1d196629de..00000000000 --- a/salt/ext/tornado/options.py +++ /dev/null @@ -1,595 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A command line parsing module that lets modules define their own options. - -Each module defines its own options which are added to the global -option namespace, e.g.:: - - from salt.ext.tornado.options import define, options - - define("mysql_host", default="127.0.0.1:3306", help="Main user DB") - define("memcache_hosts", default="127.0.0.1:11011", multiple=True, - help="Main user memcache servers") - - def connect(): - db = database.Connection(options.mysql_host) - ... - -The ``main()`` method of your application does not need to be aware of all of -the options used throughout your program; they are all automatically loaded -when the modules are loaded. However, all modules that define options -must have been imported before the command line is parsed. - -Your ``main()`` method can parse the command line or parse a config file with -either:: - - tornado.options.parse_command_line() - # or - tornado.options.parse_config_file("/etc/server.conf") - -.. note: - - When using tornado.options.parse_command_line or - tornado.options.parse_config_file, the only options that are set are - ones that were previously defined with tornado.options.define. - -Command line formats are what you would expect (``--myoption=myvalue``). -Config files are just Python files. Global names become options, e.g.:: - - myoption = "myvalue" - myotheroption = "myothervalue" - -We support `datetimes `, `timedeltas -`, ints, and floats (just pass a ``type`` kwarg to -`define`). We also accept multi-value options. See the documentation for -`define()` below. - -`tornado.options.options` is a singleton instance of `OptionParser`, and -the top-level functions in this module (`define`, `parse_command_line`, etc) -simply call methods on it. You may create additional `OptionParser` -instances to define isolated sets of options, such as for subcommands. - -.. note:: - - By default, several options are defined that will configure the - standard `logging` module when `parse_command_line` or `parse_config_file` - are called. If you want Tornado to leave the logging configuration - alone so you can manage it yourself, either pass ``--logging=none`` - on the command line or do the following to disable it in code:: - - from salt.ext.tornado.options import options, parse_command_line - options.logging = None - parse_command_line() - -.. versionchanged:: 4.3 - Dashes and underscores are fully interchangeable in option names; - options can be defined, set, and read with any mix of the two. - Dashes are typical for command-line usage while config files require - underscores. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import datetime -import numbers -import re -import sys -import os -import textwrap - -from salt.ext.tornado.escape import _unicode, native_str -from salt.ext.tornado.log import define_logging_options -from salt.ext.tornado import stack_context -from salt.ext.tornado.util import basestring_type, exec_in - - -class Error(Exception): - """Exception raised by errors in the options module.""" - pass - - -class OptionParser(object): - """A collection of options, a dictionary with object-like access. - - Normally accessed via static functions in the `tornado.options` module, - which reference a global instance. - """ - def __init__(self): - # we have to use self.__dict__ because we override setattr. - self.__dict__['_options'] = {} - self.__dict__['_parse_callbacks'] = [] - self.define("help", type=bool, help="show this help information", - callback=self._help_callback) - - def _normalize_name(self, name): - return name.replace('_', '-') - - def __getattr__(self, name): - name = self._normalize_name(name) - if isinstance(self._options.get(name), _Option): - return self._options[name].value() - raise AttributeError("Unrecognized option %r" % name) - - def __setattr__(self, name, value): - name = self._normalize_name(name) - if isinstance(self._options.get(name), _Option): - return self._options[name].set(value) - raise AttributeError("Unrecognized option %r" % name) - - def __iter__(self): - return (opt.name for opt in self._options.values()) - - def __contains__(self, name): - name = self._normalize_name(name) - return name in self._options - - def __getitem__(self, name): - return self.__getattr__(name) - - def __setitem__(self, name, value): - return self.__setattr__(name, value) - - def items(self): - """A sequence of (name, value) pairs. - - .. versionadded:: 3.1 - """ - return [(opt.name, opt.value()) for name, opt in self._options.items()] - - def groups(self): - """The set of option-groups created by ``define``. - - .. versionadded:: 3.1 - """ - return set(opt.group_name for opt in self._options.values()) - - def group_dict(self, group): - """The names and values of options in a group. - - Useful for copying options into Application settings:: - - from salt.ext.tornado.options import define, parse_command_line, options - - define('template_path', group='application') - define('static_path', group='application') - - parse_command_line() - - application = Application( - handlers, **options.group_dict('application')) - - .. versionadded:: 3.1 - """ - return dict( - (opt.name, opt.value()) for name, opt in self._options.items() - if not group or group == opt.group_name) - - def as_dict(self): - """The names and values of all options. - - .. versionadded:: 3.1 - """ - return dict( - (opt.name, opt.value()) for name, opt in self._options.items()) - - def define(self, name, default=None, type=None, help=None, metavar=None, - multiple=False, group=None, callback=None): - """Defines a new command line option. - - If ``type`` is given (one of str, float, int, datetime, or timedelta) - or can be inferred from the ``default``, we parse the command line - arguments based on the given type. If ``multiple`` is True, we accept - comma-separated values, and the option value is always a list. - - For multi-value integers, we also accept the syntax ``x:y``, which - turns into ``range(x, y)`` - very useful for long integer ranges. - - ``help`` and ``metavar`` are used to construct the - automatically generated command line help string. The help - message is formatted like:: - - --name=METAVAR help string - - ``group`` is used to group the defined options in logical - groups. By default, command line options are grouped by the - file in which they are defined. - - Command line option names must be unique globally. They can be parsed - from the command line with `parse_command_line` or parsed from a - config file with `parse_config_file`. - - If a ``callback`` is given, it will be run with the new value whenever - the option is changed. This can be used to combine command-line - and file-based options:: - - define("config", type=str, help="path to config file", - callback=lambda path: parse_config_file(path, final=False)) - - With this definition, options in the file specified by ``--config`` will - override options set earlier on the command line, but can be overridden - by later flags. - """ - normalized = self._normalize_name(name) - if normalized in self._options: - raise Error("Option %r already defined in %s" % - (normalized, self._options[normalized].file_name)) - frame = sys._getframe(0) - options_file = frame.f_code.co_filename - - # Can be called directly, or through top level define() fn, in which - # case, step up above that frame to look for real caller. - if (frame.f_back.f_code.co_filename == options_file and - frame.f_back.f_code.co_name == 'define'): - frame = frame.f_back - - file_name = frame.f_back.f_code.co_filename - if file_name == options_file: - file_name = "" - if type is None: - if not multiple and default is not None: - type = default.__class__ - else: - type = str - if group: - group_name = group - else: - group_name = file_name - option = _Option(name, file_name=file_name, - default=default, type=type, help=help, - metavar=metavar, multiple=multiple, - group_name=group_name, - callback=callback) - self._options[normalized] = option - - def parse_command_line(self, args=None, final=True): - """Parses all options given on the command line (defaults to - `sys.argv`). - - Note that ``args[0]`` is ignored since it is the program name - in `sys.argv`. - - We return a list of all arguments that are not parsed as options. - - If ``final`` is ``False``, parse callbacks will not be run. - This is useful for applications that wish to combine configurations - from multiple sources. - """ - if args is None: - args = sys.argv - remaining = [] - for i in range(1, len(args)): - # All things after the last option are command line arguments - if not args[i].startswith("-"): - remaining = args[i:] - break - if args[i] == "--": - remaining = args[i + 1:] - break - arg = args[i].lstrip("-") - name, equals, value = arg.partition("=") - name = self._normalize_name(name) - if name not in self._options: - self.print_help() - raise Error('Unrecognized command line option: %r' % name) - option = self._options[name] - if not equals: - if option.type == bool: - value = "true" - else: - raise Error('Option %r requires a value' % name) - option.parse(value) - - if final: - self.run_parse_callbacks() - - return remaining - - def parse_config_file(self, path, final=True): - """Parses and loads the Python config file at the given path. - - If ``final`` is ``False``, parse callbacks will not be run. - This is useful for applications that wish to combine configurations - from multiple sources. - - .. versionchanged:: 4.1 - Config files are now always interpreted as utf-8 instead of - the system default encoding. - - .. versionchanged:: 4.4 - The special variable ``__file__`` is available inside config - files, specifying the absolute path to the config file itself. - """ - config = {'__file__': os.path.abspath(path)} - with open(path, 'rb') as f: - exec_in(native_str(f.read()), config, config) - for name in config: - normalized = self._normalize_name(name) - if normalized in self._options: - self._options[normalized].set(config[name]) - - if final: - self.run_parse_callbacks() - - def print_help(self, file=None): - """Prints all the command line options to stderr (or another file).""" - if file is None: - file = sys.stderr - print("Usage: %s [OPTIONS]" % sys.argv[0], file=file) - print("\nOptions:\n", file=file) - by_group = {} - for option in self._options.values(): - by_group.setdefault(option.group_name, []).append(option) - - for filename, o in sorted(by_group.items()): - if filename: - print("\n%s options:\n" % os.path.normpath(filename), file=file) - o.sort(key=lambda option: option.name) - for option in o: - # Always print names with dashes in a CLI context. - prefix = self._normalize_name(option.name) - if option.metavar: - prefix += "=" + option.metavar - description = option.help or "" - if option.default is not None and option.default != '': - description += " (default %s)" % option.default - lines = textwrap.wrap(description, 79 - 35) - if len(prefix) > 30 or len(lines) == 0: - lines.insert(0, '') - print(" --%-30s %s" % (prefix, lines[0]), file=file) - for line in lines[1:]: - print("%-34s %s" % (' ', line), file=file) - print(file=file) - - def _help_callback(self, value): - if value: - self.print_help() - sys.exit(0) - - def add_parse_callback(self, callback): - """Adds a parse callback, to be invoked when option parsing is done.""" - self._parse_callbacks.append(stack_context.wrap(callback)) - - def run_parse_callbacks(self): - for callback in self._parse_callbacks: - callback() - - def mockable(self): - """Returns a wrapper around self that is compatible with - `mock.patch `. - - The `mock.patch ` function (included in - the standard library `unittest.mock` package since Python 3.3, - or in the third-party ``mock`` package for older versions of - Python) is incompatible with objects like ``options`` that - override ``__getattr__`` and ``__setattr__``. This function - returns an object that can be used with `mock.patch.object - ` to modify option values:: - - with mock.patch.object(options.mockable(), 'name', value): - assert options.name == value - """ - return _Mockable(self) - - -class _Mockable(object): - """`mock.patch` compatible wrapper for `OptionParser`. - - As of ``mock`` version 1.0.1, when an object uses ``__getattr__`` - hooks instead of ``__dict__``, ``patch.__exit__`` tries to delete - the attribute it set instead of setting a new one (assuming that - the object does not catpure ``__setattr__``, so the patch - created a new attribute in ``__dict__``). - - _Mockable's getattr and setattr pass through to the underlying - OptionParser, and delattr undoes the effect of a previous setattr. - """ - def __init__(self, options): - # Modify __dict__ directly to bypass __setattr__ - self.__dict__['_options'] = options - self.__dict__['_originals'] = {} - - def __getattr__(self, name): - return getattr(self._options, name) - - def __setattr__(self, name, value): - assert name not in self._originals, "don't reuse mockable objects" - self._originals[name] = getattr(self._options, name) - setattr(self._options, name, value) - - def __delattr__(self, name): - setattr(self._options, name, self._originals.pop(name)) - - -class _Option(object): - UNSET = object() - - def __init__(self, name, default=None, type=basestring_type, help=None, - metavar=None, multiple=False, file_name=None, group_name=None, - callback=None): - if default is None and multiple: - default = [] - self.name = name - self.type = type - self.help = help - self.metavar = metavar - self.multiple = multiple - self.file_name = file_name - self.group_name = group_name - self.callback = callback - self.default = default - self._value = _Option.UNSET - - def value(self): - return self.default if self._value is _Option.UNSET else self._value - - def parse(self, value): - _parse = { - datetime.datetime: self._parse_datetime, - datetime.timedelta: self._parse_timedelta, - bool: self._parse_bool, - basestring_type: self._parse_string, - }.get(self.type, self.type) - if self.multiple: - self._value = [] - for part in value.split(","): - if issubclass(self.type, numbers.Integral): - # allow ranges of the form X:Y (inclusive at both ends) - lo, _, hi = part.partition(":") - lo = _parse(lo) - hi = _parse(hi) if hi else lo - self._value.extend(range(lo, hi + 1)) - else: - self._value.append(_parse(part)) - else: - self._value = _parse(value) - if self.callback is not None: - self.callback(self._value) - return self.value() - - def set(self, value): - if self.multiple: - if not isinstance(value, list): - raise Error("Option %r is required to be a list of %s" % - (self.name, self.type.__name__)) - for item in value: - if item is not None and not isinstance(item, self.type): - raise Error("Option %r is required to be a list of %s" % - (self.name, self.type.__name__)) - else: - if value is not None and not isinstance(value, self.type): - raise Error("Option %r is required to be a %s (%s given)" % - (self.name, self.type.__name__, type(value))) - self._value = value - if self.callback is not None: - self.callback(self._value) - - # Supported date/time formats in our options - _DATETIME_FORMATS = [ - "%a %b %d %H:%M:%S %Y", - "%Y-%m-%d %H:%M:%S", - "%Y-%m-%d %H:%M", - "%Y-%m-%dT%H:%M", - "%Y%m%d %H:%M:%S", - "%Y%m%d %H:%M", - "%Y-%m-%d", - "%Y%m%d", - "%H:%M:%S", - "%H:%M", - ] - - def _parse_datetime(self, value): - for format in self._DATETIME_FORMATS: - try: - return datetime.datetime.strptime(value, format) - except ValueError: - pass - raise Error('Unrecognized date/time format: %r' % value) - - _TIMEDELTA_ABBREV_DICT = { - 'h': 'hours', - 'm': 'minutes', - 'min': 'minutes', - 's': 'seconds', - 'sec': 'seconds', - 'ms': 'milliseconds', - 'us': 'microseconds', - 'd': 'days', - 'w': 'weeks', - } - - _FLOAT_PATTERN = r'[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?' - - _TIMEDELTA_PATTERN = re.compile( - r'\s*(%s)\s*(\w*)\s*' % _FLOAT_PATTERN, re.IGNORECASE) - - def _parse_timedelta(self, value): - try: - sum = datetime.timedelta() - start = 0 - while start < len(value): - m = self._TIMEDELTA_PATTERN.match(value, start) - if not m: - raise Exception() - num = float(m.group(1)) - units = m.group(2) or 'seconds' - units = self._TIMEDELTA_ABBREV_DICT.get(units, units) - sum += datetime.timedelta(**{units: num}) - start = m.end() - return sum - except Exception: - raise - - def _parse_bool(self, value): - return value.lower() not in ("false", "0", "f") - - def _parse_string(self, value): - return _unicode(value) - - -options = OptionParser() -"""Global options object. - -All defined options are available as attributes on this object. -""" - - -def define(name, default=None, type=None, help=None, metavar=None, - multiple=False, group=None, callback=None): - """Defines an option in the global namespace. - - See `OptionParser.define`. - """ - return options.define(name, default=default, type=type, help=help, - metavar=metavar, multiple=multiple, group=group, - callback=callback) - - -def parse_command_line(args=None, final=True): - """Parses global options from the command line. - - See `OptionParser.parse_command_line`. - """ - return options.parse_command_line(args, final=final) - - -def parse_config_file(path, final=True): - """Parses global options from a config file. - - See `OptionParser.parse_config_file`. - """ - return options.parse_config_file(path, final=final) - - -def print_help(file=None): - """Prints all the command line options to stderr (or another file). - - See `OptionParser.print_help`. - """ - return options.print_help(file) - - -def add_parse_callback(callback): - """Adds a parse callback, to be invoked when option parsing is done. - - See `OptionParser.add_parse_callback` - """ - options.add_parse_callback(callback) - - -# Default options -define_logging_options(options) diff --git a/salt/ext/tornado/platform/__init__.py b/salt/ext/tornado/platform/__init__.py deleted file mode 100644 index 388083ed935..00000000000 --- a/salt/ext/tornado/platform/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# pylint: skip-file diff --git a/salt/ext/tornado/platform/asyncio.py b/salt/ext/tornado/platform/asyncio.py deleted file mode 100644 index 07b5000c2e5..00000000000 --- a/salt/ext/tornado/platform/asyncio.py +++ /dev/null @@ -1,223 +0,0 @@ -"""Bridges between the `asyncio` module and Tornado IOLoop. - -.. versionadded:: 3.2 - -This module integrates Tornado with the ``asyncio`` module introduced -in Python 3.4 (and available `as a separate download -`_ for Python 3.3). This makes -it possible to combine the two libraries on the same event loop. - -Most applications should use `AsyncIOMainLoop` to run Tornado on the -default ``asyncio`` event loop. Applications that need to run event -loops on multiple threads may use `AsyncIOLoop` to create multiple -loops. - -.. note:: - - Tornado requires the `~asyncio.AbstractEventLoop.add_reader` family of - methods, so it is not compatible with the `~asyncio.ProactorEventLoop` on - Windows. Use the `~asyncio.SelectorEventLoop` instead. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function -import functools - -import salt.ext.tornado.concurrent -from salt.ext.tornado.gen import convert_yielded -from salt.ext.tornado.ioloop import IOLoop -from salt.ext.tornado import stack_context - -try: - # Import the real asyncio module for py33+ first. Older versions of the - # trollius backport also use this name. - import asyncio # type: ignore -except ImportError as e: - # Asyncio itself isn't available; see if trollius is (backport to py26+). - try: - import trollius as asyncio # type: ignore - except ImportError: - # Re-raise the original asyncio error, not the trollius one. - raise e - - -class BaseAsyncIOLoop(IOLoop): - def initialize(self, asyncio_loop, close_loop=False, **kwargs): - super(BaseAsyncIOLoop, self).initialize(**kwargs) - self.asyncio_loop = asyncio_loop - self.close_loop = close_loop - # Maps fd to (fileobj, handler function) pair (as in IOLoop.add_handler) - self.handlers = {} - # Set of fds listening for reads/writes - self.readers = set() - self.writers = set() - self.closing = False - - def close(self, all_fds=False): - self.closing = True - for fd in list(self.handlers): - fileobj, handler_func = self.handlers[fd] - self.remove_handler(fd) - if all_fds: - self.close_fd(fileobj) - if self.close_loop: - self.asyncio_loop.close() - - def add_handler(self, fd, handler, events): - fd, fileobj = self.split_fd(fd) - if fd in self.handlers: - raise ValueError("fd %s added twice" % fd) - self.handlers[fd] = (fileobj, stack_context.wrap(handler)) - if events & IOLoop.READ: - self.asyncio_loop.add_reader( - fd, self._handle_events, fd, IOLoop.READ) - self.readers.add(fd) - if events & IOLoop.WRITE: - self.asyncio_loop.add_writer( - fd, self._handle_events, fd, IOLoop.WRITE) - self.writers.add(fd) - - def update_handler(self, fd, events): - fd, fileobj = self.split_fd(fd) - if events & IOLoop.READ: - if fd not in self.readers: - self.asyncio_loop.add_reader( - fd, self._handle_events, fd, IOLoop.READ) - self.readers.add(fd) - else: - if fd in self.readers: - self.asyncio_loop.remove_reader(fd) - self.readers.remove(fd) - if events & IOLoop.WRITE: - if fd not in self.writers: - self.asyncio_loop.add_writer( - fd, self._handle_events, fd, IOLoop.WRITE) - self.writers.add(fd) - else: - if fd in self.writers: - self.asyncio_loop.remove_writer(fd) - self.writers.remove(fd) - - def remove_handler(self, fd): - fd, fileobj = self.split_fd(fd) - if fd not in self.handlers: - return - if fd in self.readers: - self.asyncio_loop.remove_reader(fd) - self.readers.remove(fd) - if fd in self.writers: - self.asyncio_loop.remove_writer(fd) - self.writers.remove(fd) - del self.handlers[fd] - - def _handle_events(self, fd, events): - fileobj, handler_func = self.handlers[fd] - handler_func(fileobj, events) - - def start(self): - old_current = IOLoop.current(instance=False) - try: - self._setup_logging() - self.make_current() - self.asyncio_loop.run_forever() - finally: - if old_current is None: - IOLoop.clear_current() - else: - old_current.make_current() - - def stop(self): - self.asyncio_loop.stop() - - def call_at(self, when, callback, *args, **kwargs): - # asyncio.call_at supports *args but not **kwargs, so bind them here. - # We do not synchronize self.time and asyncio_loop.time, so - # convert from absolute to relative. - return self.asyncio_loop.call_later( - max(0, when - self.time()), self._run_callback, - functools.partial(stack_context.wrap(callback), *args, **kwargs)) - - def remove_timeout(self, timeout): - timeout.cancel() - - def add_callback(self, callback, *args, **kwargs): - if self.closing: - # TODO: this is racy; we need a lock to ensure that the - # loop isn't closed during call_soon_threadsafe. - raise RuntimeError("IOLoop is closing") - self.asyncio_loop.call_soon_threadsafe( - self._run_callback, - functools.partial(stack_context.wrap(callback), *args, **kwargs)) - - add_callback_from_signal = add_callback - - -class AsyncIOMainLoop(BaseAsyncIOLoop): - """``AsyncIOMainLoop`` creates an `.IOLoop` that corresponds to the - current ``asyncio`` event loop (i.e. the one returned by - ``asyncio.get_event_loop()``). Recommended usage:: - - from salt.ext.tornado.platform.asyncio import AsyncIOMainLoop - import asyncio - AsyncIOMainLoop().install() - asyncio.get_event_loop().run_forever() - - See also :meth:`tornado.ioloop.IOLoop.install` for general notes on - installing alternative IOLoops. - """ - def initialize(self, **kwargs): - super(AsyncIOMainLoop, self).initialize(asyncio.get_event_loop(), - close_loop=False, **kwargs) - - -class AsyncIOLoop(BaseAsyncIOLoop): - """``AsyncIOLoop`` is an `.IOLoop` that runs on an ``asyncio`` event loop. - This class follows the usual Tornado semantics for creating new - ``IOLoops``; these loops are not necessarily related to the - ``asyncio`` default event loop. Recommended usage:: - - from salt.ext.tornado.ioloop import IOLoop - IOLoop.configure('tornado.platform.asyncio.AsyncIOLoop') - IOLoop.current().start() - - Each ``AsyncIOLoop`` creates a new ``asyncio.EventLoop``; this object - can be accessed with the ``asyncio_loop`` attribute. - """ - def initialize(self, **kwargs): - loop = asyncio.new_event_loop() - try: - super(AsyncIOLoop, self).initialize(loop, close_loop=True, **kwargs) - except Exception: - # If initialize() does not succeed (taking ownership of the loop), - # we have to close it. - loop.close() - raise - - -def to_tornado_future(asyncio_future): - """Convert an `asyncio.Future` to a `tornado.concurrent.Future`. - - .. versionadded:: 4.1 - """ - tf = salt.ext.tornado.concurrent.Future() - salt.ext.tornado.concurrent.chain_future(asyncio_future, tf) - return tf - - -def to_asyncio_future(tornado_future): - """Convert a Tornado yieldable object to an `asyncio.Future`. - - .. versionadded:: 4.1 - - .. versionchanged:: 4.3 - Now accepts any yieldable object, not just - `tornado.concurrent.Future`. - """ - tornado_future = convert_yielded(tornado_future) - af = asyncio.Future() - salt.ext.tornado.concurrent.chain_future(tornado_future, af) - return af - - -if hasattr(convert_yielded, 'register'): - convert_yielded.register(asyncio.Future, to_tornado_future) # type: ignore diff --git a/salt/ext/tornado/platform/auto.py b/salt/ext/tornado/platform/auto.py deleted file mode 100644 index c66404d14ad..00000000000 --- a/salt/ext/tornado/platform/auto.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2011 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Implementation of platform-specific functionality. - -For each function or class described in `tornado.platform.interface`, -the appropriate platform-specific implementation exists in this module. -Most code that needs access to this functionality should do e.g.:: - - from salt.ext.tornado.platform.auto import set_close_exec -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import os - -if 'APPENGINE_RUNTIME' in os.environ: - from salt.ext.tornado.platform.common import Waker - - def set_close_exec(fd): - pass -elif os.name == 'nt': - from salt.ext.tornado.platform.common import Waker - from salt.ext.tornado.platform.windows import set_close_exec -else: - from salt.ext.tornado.platform.posix import set_close_exec, Waker - -try: - # monotime monkey-patches the time module to have a monotonic function - # in versions of python before 3.3. - import monotime - # Silence pyflakes warning about this unused import - monotime -except ImportError: - pass -try: - # monotonic can provide a monotonic function in versions of python before - # 3.3, too. - from monotonic import monotonic as monotonic_time -except ImportError: - try: - from time import monotonic as monotonic_time - except ImportError: - monotonic_time = None - -__all__ = ['Waker', 'set_close_exec', 'monotonic_time'] diff --git a/salt/ext/tornado/platform/auto.pyi b/salt/ext/tornado/platform/auto.pyi deleted file mode 100644 index a1c97228a43..00000000000 --- a/salt/ext/tornado/platform/auto.pyi +++ /dev/null @@ -1,4 +0,0 @@ -# auto.py is full of patterns mypy doesn't like, so for type checking -# purposes we replace it with interface.py. - -from .interface import * diff --git a/salt/ext/tornado/platform/caresresolver.py b/salt/ext/tornado/platform/caresresolver.py deleted file mode 100644 index e72868e5ccc..00000000000 --- a/salt/ext/tornado/platform/caresresolver.py +++ /dev/null @@ -1,80 +0,0 @@ -# pylint: skip-file -from __future__ import absolute_import, division, print_function -import pycares # type: ignore -import socket - -from salt.ext.tornado import gen -from salt.ext.tornado.ioloop import IOLoop -from salt.ext.tornado.netutil import Resolver, is_valid_ip - - -class CaresResolver(Resolver): - """Name resolver based on the c-ares library. - - This is a non-blocking and non-threaded resolver. It may not produce - the same results as the system resolver, but can be used for non-blocking - resolution when threads cannot be used. - - c-ares fails to resolve some names when ``family`` is ``AF_UNSPEC``, - so it is only recommended for use in ``AF_INET`` (i.e. IPv4). This is - the default for ``tornado.simple_httpclient``, but other libraries - may default to ``AF_UNSPEC``. - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - """ - def initialize(self, io_loop=None): - self.io_loop = io_loop or IOLoop.current() - self.channel = pycares.Channel(sock_state_cb=self._sock_state_cb) - self.fds = {} - - def _sock_state_cb(self, fd, readable, writable): - state = ((IOLoop.READ if readable else 0) | - (IOLoop.WRITE if writable else 0)) - if not state: - self.io_loop.remove_handler(fd) - del self.fds[fd] - elif fd in self.fds: - self.io_loop.update_handler(fd, state) - self.fds[fd] = state - else: - self.io_loop.add_handler(fd, self._handle_events, state) - self.fds[fd] = state - - def _handle_events(self, fd, events): - read_fd = pycares.ARES_SOCKET_BAD - write_fd = pycares.ARES_SOCKET_BAD - if events & IOLoop.READ: - read_fd = fd - if events & IOLoop.WRITE: - write_fd = fd - self.channel.process_fd(read_fd, write_fd) - - @gen.coroutine - def resolve(self, host, port, family=0): - if is_valid_ip(host): - addresses = [host] - else: - # gethostbyname doesn't take callback as a kwarg - self.channel.gethostbyname(host, family, (yield gen.Callback(1))) - callback_args = yield gen.Wait(1) - assert isinstance(callback_args, gen.Arguments) - assert not callback_args.kwargs - result, error = callback_args.args - if error: - raise IOError('C-Ares returned error %s: %s while resolving %s' % - (error, pycares.errno.strerror(error), host)) - addresses = result.addresses - addrinfo = [] - for address in addresses: - if '.' in address: - address_family = socket.AF_INET - elif ':' in address: - address_family = socket.AF_INET6 - else: - address_family = socket.AF_UNSPEC - if family != socket.AF_UNSPEC and family != address_family: - raise IOError('Requested socket family %d but got %d' % - (family, address_family)) - addrinfo.append((address_family, (address, port))) - raise gen.Return(addrinfo) diff --git a/salt/ext/tornado/platform/common.py b/salt/ext/tornado/platform/common.py deleted file mode 100644 index 3a265a1356e..00000000000 --- a/salt/ext/tornado/platform/common.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Lowest-common-denominator implementations of platform functionality.""" -# pylint: skip-file -from __future__ import absolute_import, division, print_function - -import errno -import socket -import time - -from salt.ext.tornado.platform import interface -from salt.ext.tornado.util import errno_from_exception - - -def try_close(f): - # Avoid issue #875 (race condition when using the file in another - # thread). - for i in range(10): - try: - f.close() - except IOError: - # Yield to another thread - time.sleep(1e-3) - else: - break - # Try a last time and let raise - f.close() - - -class Waker(interface.Waker): - """Create an OS independent asynchronous pipe. - - For use on platforms that don't have os.pipe() (or where pipes cannot - be passed to select()), but do have sockets. This includes Windows - and Jython. - """ - def __init__(self): - from .auto import set_close_exec - # Based on Zope select_trigger.py: - # https://github.com/zopefoundation/Zope/blob/master/src/ZServer/medusa/thread/select_trigger.py - - self.writer = socket.socket() - set_close_exec(self.writer.fileno()) - # Disable buffering -- pulling the trigger sends 1 byte, - # and we want that sent immediately, to wake up ASAP. - self.writer.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - - count = 0 - while 1: - count += 1 - # Bind to a local port; for efficiency, let the OS pick - # a free port for us. - # Unfortunately, stress tests showed that we may not - # be able to connect to that port ("Address already in - # use") despite that the OS picked it. This appears - # to be a race bug in the Windows socket implementation. - # So we loop until a connect() succeeds (almost always - # on the first try). See the long thread at - # http://mail.zope.org/pipermail/zope/2005-July/160433.html - # for hideous details. - a = socket.socket() - set_close_exec(a.fileno()) - a.bind(("127.0.0.1", 0)) - a.listen(1) - connect_address = a.getsockname() # assigned (host, port) pair - try: - self.writer.connect(connect_address) - break # success - except socket.error as detail: - if (not hasattr(errno, 'WSAEADDRINUSE') or - errno_from_exception(detail) != errno.WSAEADDRINUSE): - # "Address already in use" is the only error - # I've seen on two WinXP Pro SP2 boxes, under - # Pythons 2.3.5 and 2.4.1. - raise - # (10048, 'Address already in use') - # assert count <= 2 # never triggered in Tim's tests - if count >= 10: # I've never seen it go above 2 - a.close() - self.writer.close() - raise socket.error("Cannot bind trigger!") - # Close `a` and try again. Note: I originally put a short - # sleep() here, but it didn't appear to help or hurt. - a.close() - - self.reader, addr = a.accept() - set_close_exec(self.reader.fileno()) - self.reader.setblocking(0) - self.writer.setblocking(0) - a.close() - self.reader_fd = self.reader.fileno() - - def fileno(self): - return self.reader.fileno() - - def write_fileno(self): - return self.writer.fileno() - - def wake(self): - try: - self.writer.send(b"x") - except (IOError, socket.error, ValueError): - pass - - def consume(self): - try: - while True: - result = self.reader.recv(1024) - if not result: - break - except (IOError, socket.error): - pass - - def close(self): - self.reader.close() - try_close(self.writer) diff --git a/salt/ext/tornado/platform/epoll.py b/salt/ext/tornado/platform/epoll.py deleted file mode 100644 index a6bea37a17e..00000000000 --- a/salt/ext/tornado/platform/epoll.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# pylint: skip-file -"""EPoll-based IOLoop implementation for Linux systems.""" -from __future__ import absolute_import, division, print_function - -import select - -from salt.ext.tornado.ioloop import PollIOLoop - - -class EPollIOLoop(PollIOLoop): - def initialize(self, **kwargs): - super(EPollIOLoop, self).initialize(impl=select.epoll(), **kwargs) diff --git a/salt/ext/tornado/platform/interface.py b/salt/ext/tornado/platform/interface.py deleted file mode 100644 index 444c18312e3..00000000000 --- a/salt/ext/tornado/platform/interface.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2011 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Interfaces for platform-specific functionality. - -This module exists primarily for documentation purposes and as base classes -for other tornado.platform modules. Most code should import the appropriate -implementation from `tornado.platform.auto`. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - - -def set_close_exec(fd): - """Sets the close-on-exec bit (``FD_CLOEXEC``)for a file descriptor.""" - raise NotImplementedError() - - -class Waker(object): - """A socket-like object that can wake another thread from ``select()``. - - The `~tornado.ioloop.IOLoop` will add the Waker's `fileno()` to - its ``select`` (or ``epoll`` or ``kqueue``) calls. When another - thread wants to wake up the loop, it calls `wake`. Once it has woken - up, it will call `consume` to do any necessary per-wake cleanup. When - the ``IOLoop`` is closed, it closes its waker too. - """ - def fileno(self): - """Returns the read file descriptor for this waker. - - Must be suitable for use with ``select()`` or equivalent on the - local platform. - """ - raise NotImplementedError() - - def write_fileno(self): - """Returns the write file descriptor for this waker.""" - raise NotImplementedError() - - def wake(self): - """Triggers activity on the waker's file descriptor.""" - raise NotImplementedError() - - def consume(self): - """Called after the listen has woken up to do any necessary cleanup.""" - raise NotImplementedError() - - def close(self): - """Closes the waker's file descriptor(s).""" - raise NotImplementedError() - - -def monotonic_time(): - raise NotImplementedError() diff --git a/salt/ext/tornado/platform/kqueue.py b/salt/ext/tornado/platform/kqueue.py deleted file mode 100644 index 65aeeec9f42..00000000000 --- a/salt/ext/tornado/platform/kqueue.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""KQueue-based IOLoop implementation for BSD/Mac systems.""" -# pylint: skip-file -from __future__ import absolute_import, division, print_function - -import select - -from salt.ext.tornado.ioloop import IOLoop, PollIOLoop - -assert hasattr(select, 'kqueue'), 'kqueue not supported' - - -class _KQueue(object): - """A kqueue-based event loop for BSD/Mac systems.""" - def __init__(self): - self._kqueue = select.kqueue() - self._active = {} - - def fileno(self): - return self._kqueue.fileno() - - def close(self): - self._kqueue.close() - - def register(self, fd, events): - if fd in self._active: - raise IOError("fd %s already registered" % fd) - self._control(fd, events, select.KQ_EV_ADD) - self._active[fd] = events - - def modify(self, fd, events): - self.unregister(fd) - self.register(fd, events) - - def unregister(self, fd): - events = self._active.pop(fd) - self._control(fd, events, select.KQ_EV_DELETE) - - def _control(self, fd, events, flags): - kevents = [] - if events & IOLoop.WRITE: - kevents.append(select.kevent( - fd, filter=select.KQ_FILTER_WRITE, flags=flags)) - if events & IOLoop.READ: - kevents.append(select.kevent( - fd, filter=select.KQ_FILTER_READ, flags=flags)) - # Even though control() takes a list, it seems to return EINVAL - # on Mac OS X (10.6) when there is more than one event in the list. - for kevent in kevents: - self._kqueue.control([kevent], 0) - - def poll(self, timeout): - kevents = self._kqueue.control(None, 1000, timeout) - events = {} - for kevent in kevents: - fd = kevent.ident - if kevent.filter == select.KQ_FILTER_READ: - events[fd] = events.get(fd, 0) | IOLoop.READ - if kevent.filter == select.KQ_FILTER_WRITE: - if kevent.flags & select.KQ_EV_EOF: - # If an asynchronous connection is refused, kqueue - # returns a write event with the EOF flag set. - # Turn this into an error for consistency with the - # other IOLoop implementations. - # Note that for read events, EOF may be returned before - # all data has been consumed from the socket buffer, - # so we only check for EOF on write events. - events[fd] = IOLoop.ERROR - else: - events[fd] = events.get(fd, 0) | IOLoop.WRITE - if kevent.flags & select.KQ_EV_ERROR: - events[fd] = events.get(fd, 0) | IOLoop.ERROR - return events.items() - - -class KQueueIOLoop(PollIOLoop): - def initialize(self, **kwargs): - super(KQueueIOLoop, self).initialize(impl=_KQueue(), **kwargs) diff --git a/salt/ext/tornado/platform/posix.py b/salt/ext/tornado/platform/posix.py deleted file mode 100644 index fa5c02677e9..00000000000 --- a/salt/ext/tornado/platform/posix.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2011 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Posix implementations of platform-specific functionality.""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import fcntl -import os - -from salt.ext.tornado.platform import common, interface - - -def set_close_exec(fd): - flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) - - -def _set_nonblocking(fd): - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) - - -class Waker(interface.Waker): - def __init__(self): - r, w = os.pipe() - _set_nonblocking(r) - _set_nonblocking(w) - set_close_exec(r) - set_close_exec(w) - self.reader = os.fdopen(r, "rb", 0) - self.writer = os.fdopen(w, "wb", 0) - - def fileno(self): - return self.reader.fileno() - - def write_fileno(self): - return self.writer.fileno() - - def wake(self): - try: - self.writer.write(b"x") - except (IOError, ValueError): - pass - - def consume(self): - try: - while True: - result = self.reader.read() - if not result: - break - except IOError: - pass - - def close(self): - self.reader.close() - common.try_close(self.writer) diff --git a/salt/ext/tornado/platform/select.py b/salt/ext/tornado/platform/select.py deleted file mode 100644 index 8d91d6261fe..00000000000 --- a/salt/ext/tornado/platform/select.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Select-based IOLoop implementation. - -Used as a fallback for systems that don't support epoll or kqueue. -""" -# pylint: skip-file -from __future__ import absolute_import, division, print_function - -import select - -from salt.ext.tornado.ioloop import IOLoop, PollIOLoop - - -class _Select(object): - """A simple, select()-based IOLoop implementation for non-Linux systems""" - def __init__(self): - self.read_fds = set() - self.write_fds = set() - self.error_fds = set() - self.fd_sets = (self.read_fds, self.write_fds, self.error_fds) - - def close(self): - pass - - def register(self, fd, events): - if fd in self.read_fds or fd in self.write_fds or fd in self.error_fds: - raise IOError("fd %s already registered" % fd) - if events & IOLoop.READ: - self.read_fds.add(fd) - if events & IOLoop.WRITE: - self.write_fds.add(fd) - if events & IOLoop.ERROR: - self.error_fds.add(fd) - # Closed connections are reported as errors by epoll and kqueue, - # but as zero-byte reads by select, so when errors are requested - # we need to listen for both read and error. - # self.read_fds.add(fd) - - def modify(self, fd, events): - self.unregister(fd) - self.register(fd, events) - - def unregister(self, fd): - self.read_fds.discard(fd) - self.write_fds.discard(fd) - self.error_fds.discard(fd) - - def poll(self, timeout): - readable, writeable, errors = select.select( - self.read_fds, self.write_fds, self.error_fds, timeout) - events = {} - for fd in readable: - events[fd] = events.get(fd, 0) | IOLoop.READ - for fd in writeable: - events[fd] = events.get(fd, 0) | IOLoop.WRITE - for fd in errors: - events[fd] = events.get(fd, 0) | IOLoop.ERROR - return events.items() - - -class SelectIOLoop(PollIOLoop): - def initialize(self, **kwargs): - super(SelectIOLoop, self).initialize(impl=_Select(), **kwargs) diff --git a/salt/ext/tornado/platform/twisted.py b/salt/ext/tornado/platform/twisted.py deleted file mode 100644 index f10875ff600..00000000000 --- a/salt/ext/tornado/platform/twisted.py +++ /dev/null @@ -1,592 +0,0 @@ -# Author: Ovidiu Predescu -# Date: July 2011 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Bridges between the Twisted reactor and Tornado IOLoop. - -This module lets you run applications and libraries written for -Twisted in a Tornado application. It can be used in two modes, -depending on which library's underlying event loop you want to use. - -This module has been tested with Twisted versions 11.0.0 and newer. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import datetime -import functools -import numbers -import socket -import sys - -import twisted.internet.abstract # type: ignore -from twisted.internet.defer import Deferred # type: ignore -from twisted.internet.posixbase import PosixReactorBase # type: ignore -from twisted.internet.interfaces import IReactorFDSet, IDelayedCall, IReactorTime, IReadDescriptor, IWriteDescriptor # type: ignore -from twisted.python import failure, log # type: ignore -from twisted.internet import error # type: ignore -import twisted.names.cache # type: ignore -import twisted.names.client # type: ignore -import twisted.names.hosts # type: ignore -import twisted.names.resolve # type: ignore - -from zope.interface import implementer # type: ignore - -from salt.ext.tornado.concurrent import Future -from salt.ext.tornado.escape import utf8 -from salt.ext.tornado import gen -import salt.ext.tornado.ioloop -from salt.ext.tornado.log import app_log -from salt.ext.tornado.netutil import Resolver -from salt.ext.tornado.stack_context import NullContext, wrap -from salt.ext.tornado.ioloop import IOLoop -from salt.ext.tornado.util import timedelta_to_seconds - - -@implementer(IDelayedCall) -class TornadoDelayedCall(object): - """DelayedCall object for Tornado.""" - def __init__(self, reactor, seconds, f, *args, **kw): - self._reactor = reactor - self._func = functools.partial(f, *args, **kw) - self._time = self._reactor.seconds() + seconds - self._timeout = self._reactor._io_loop.add_timeout(self._time, - self._called) - self._active = True - - def _called(self): - self._active = False - self._reactor._removeDelayedCall(self) - try: - self._func() - except: - app_log.error("_called caught exception", exc_info=True) - - def getTime(self): - return self._time - - def cancel(self): - self._active = False - self._reactor._io_loop.remove_timeout(self._timeout) - self._reactor._removeDelayedCall(self) - - def delay(self, seconds): - self._reactor._io_loop.remove_timeout(self._timeout) - self._time += seconds - self._timeout = self._reactor._io_loop.add_timeout(self._time, - self._called) - - def reset(self, seconds): - self._reactor._io_loop.remove_timeout(self._timeout) - self._time = self._reactor.seconds() + seconds - self._timeout = self._reactor._io_loop.add_timeout(self._time, - self._called) - - def active(self): - return self._active - - -@implementer(IReactorTime, IReactorFDSet) -class TornadoReactor(PosixReactorBase): - """Twisted reactor built on the Tornado IOLoop. - - `TornadoReactor` implements the Twisted reactor interface on top of - the Tornado IOLoop. To use it, simply call `install` at the beginning - of the application:: - - import tornado.platform.twisted - tornado.platform.twisted.install() - from twisted.internet import reactor - - When the app is ready to start, call ``IOLoop.current().start()`` - instead of ``reactor.run()``. - - It is also possible to create a non-global reactor by calling - ``tornado.platform.twisted.TornadoReactor(io_loop)``. However, if - the `.IOLoop` and reactor are to be short-lived (such as those used in - unit tests), additional cleanup may be required. Specifically, it is - recommended to call:: - - reactor.fireSystemEvent('shutdown') - reactor.disconnectAll() - - before closing the `.IOLoop`. - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - """ - def __init__(self, io_loop=None): - if not io_loop: - io_loop = salt.ext.tornado.ioloop.IOLoop.current() - self._io_loop = io_loop - self._readers = {} # map of reader objects to fd - self._writers = {} # map of writer objects to fd - self._fds = {} # a map of fd to a (reader, writer) tuple - self._delayedCalls = {} - PosixReactorBase.__init__(self) - self.addSystemEventTrigger('during', 'shutdown', self.crash) - - # IOLoop.start() bypasses some of the reactor initialization. - # Fire off the necessary events if they weren't already triggered - # by reactor.run(). - def start_if_necessary(): - if not self._started: - self.fireSystemEvent('startup') - self._io_loop.add_callback(start_if_necessary) - - # IReactorTime - def seconds(self): - return self._io_loop.time() - - def callLater(self, seconds, f, *args, **kw): - dc = TornadoDelayedCall(self, seconds, f, *args, **kw) - self._delayedCalls[dc] = True - return dc - - def getDelayedCalls(self): - return [x for x in self._delayedCalls if x._active] - - def _removeDelayedCall(self, dc): - if dc in self._delayedCalls: - del self._delayedCalls[dc] - - # IReactorThreads - def callFromThread(self, f, *args, **kw): - assert callable(f), "%s is not callable" % f - with NullContext(): - # This NullContext is mainly for an edge case when running - # TwistedIOLoop on top of a TornadoReactor. - # TwistedIOLoop.add_callback uses reactor.callFromThread and - # should not pick up additional StackContexts along the way. - self._io_loop.add_callback(f, *args, **kw) - - # We don't need the waker code from the super class, Tornado uses - # its own waker. - def installWaker(self): - pass - - def wakeUp(self): - pass - - # IReactorFDSet - def _invoke_callback(self, fd, events): - if fd not in self._fds: - return - (reader, writer) = self._fds[fd] - if reader: - err = None - if reader.fileno() == -1: - err = error.ConnectionLost() - elif events & IOLoop.READ: - err = log.callWithLogger(reader, reader.doRead) - if err is None and events & IOLoop.ERROR: - err = error.ConnectionLost() - if err is not None: - self.removeReader(reader) - reader.readConnectionLost(failure.Failure(err)) - if writer: - err = None - if writer.fileno() == -1: - err = error.ConnectionLost() - elif events & IOLoop.WRITE: - err = log.callWithLogger(writer, writer.doWrite) - if err is None and events & IOLoop.ERROR: - err = error.ConnectionLost() - if err is not None: - self.removeWriter(writer) - writer.writeConnectionLost(failure.Failure(err)) - - def addReader(self, reader): - if reader in self._readers: - # Don't add the reader if it's already there - return - fd = reader.fileno() - self._readers[reader] = fd - if fd in self._fds: - (_, writer) = self._fds[fd] - self._fds[fd] = (reader, writer) - if writer: - # We already registered this fd for write events, - # update it for read events as well. - self._io_loop.update_handler(fd, IOLoop.READ | IOLoop.WRITE) - else: - with NullContext(): - self._fds[fd] = (reader, None) - self._io_loop.add_handler(fd, self._invoke_callback, - IOLoop.READ) - - def addWriter(self, writer): - if writer in self._writers: - return - fd = writer.fileno() - self._writers[writer] = fd - if fd in self._fds: - (reader, _) = self._fds[fd] - self._fds[fd] = (reader, writer) - if reader: - # We already registered this fd for read events, - # update it for write events as well. - self._io_loop.update_handler(fd, IOLoop.READ | IOLoop.WRITE) - else: - with NullContext(): - self._fds[fd] = (None, writer) - self._io_loop.add_handler(fd, self._invoke_callback, - IOLoop.WRITE) - - def removeReader(self, reader): - if reader in self._readers: - fd = self._readers.pop(reader) - (_, writer) = self._fds[fd] - if writer: - # We have a writer so we need to update the IOLoop for - # write events only. - self._fds[fd] = (None, writer) - self._io_loop.update_handler(fd, IOLoop.WRITE) - else: - # Since we have no writer registered, we remove the - # entry from _fds and unregister the handler from the - # IOLoop - del self._fds[fd] - self._io_loop.remove_handler(fd) - - def removeWriter(self, writer): - if writer in self._writers: - fd = self._writers.pop(writer) - (reader, _) = self._fds[fd] - if reader: - # We have a reader so we need to update the IOLoop for - # read events only. - self._fds[fd] = (reader, None) - self._io_loop.update_handler(fd, IOLoop.READ) - else: - # Since we have no reader registered, we remove the - # entry from the _fds and unregister the handler from - # the IOLoop. - del self._fds[fd] - self._io_loop.remove_handler(fd) - - def removeAll(self): - return self._removeAll(self._readers, self._writers) - - def getReaders(self): - return self._readers.keys() - - def getWriters(self): - return self._writers.keys() - - # The following functions are mainly used in twisted-style test cases; - # it is expected that most users of the TornadoReactor will call - # IOLoop.start() instead of Reactor.run(). - def stop(self): - PosixReactorBase.stop(self) - fire_shutdown = functools.partial(self.fireSystemEvent, "shutdown") - self._io_loop.add_callback(fire_shutdown) - - def crash(self): - PosixReactorBase.crash(self) - self._io_loop.stop() - - def doIteration(self, delay): - raise NotImplementedError("doIteration") - - def mainLoop(self): - # Since this class is intended to be used in applications - # where the top-level event loop is ``io_loop.start()`` rather - # than ``reactor.run()``, it is implemented a little - # differently than other Twisted reactors. We override - # ``mainLoop`` instead of ``doIteration`` and must implement - # timed call functionality on top of `.IOLoop.add_timeout` - # rather than using the implementation in - # ``PosixReactorBase``. - self._io_loop.start() - - -class _TestReactor(TornadoReactor): - """Subclass of TornadoReactor for use in unittests. - - This can't go in the test.py file because of import-order dependencies - with the Twisted reactor test builder. - """ - def __init__(self): - # always use a new ioloop - super(_TestReactor, self).__init__(IOLoop()) - - def listenTCP(self, port, factory, backlog=50, interface=''): - # default to localhost to avoid firewall prompts on the mac - if not interface: - interface = '127.0.0.1' - return super(_TestReactor, self).listenTCP( - port, factory, backlog=backlog, interface=interface) - - def listenUDP(self, port, protocol, interface='', maxPacketSize=8192): - if not interface: - interface = '127.0.0.1' - return super(_TestReactor, self).listenUDP( - port, protocol, interface=interface, maxPacketSize=maxPacketSize) - - -def install(io_loop=None): - """Install this package as the default Twisted reactor. - - ``install()`` must be called very early in the startup process, - before most other twisted-related imports. Conversely, because it - initializes the `.IOLoop`, it cannot be called before - `.fork_processes` or multi-process `~.TCPServer.start`. These - conflicting requirements make it difficult to use `.TornadoReactor` - in multi-process mode, and an external process manager such as - ``supervisord`` is recommended instead. - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - - """ - if not io_loop: - io_loop = salt.ext.tornado.ioloop.IOLoop.current() - reactor = TornadoReactor(io_loop) - from twisted.internet.main import installReactor # type: ignore - installReactor(reactor) - return reactor - - -@implementer(IReadDescriptor, IWriteDescriptor) -class _FD(object): - def __init__(self, fd, fileobj, handler): - self.fd = fd - self.fileobj = fileobj - self.handler = handler - self.reading = False - self.writing = False - self.lost = False - - def fileno(self): - return self.fd - - def doRead(self): - if not self.lost: - self.handler(self.fileobj, salt.ext.tornado.ioloop.IOLoop.READ) - - def doWrite(self): - if not self.lost: - self.handler(self.fileobj, salt.ext.tornado.ioloop.IOLoop.WRITE) - - def connectionLost(self, reason): - if not self.lost: - self.handler(self.fileobj, salt.ext.tornado.ioloop.IOLoop.ERROR) - self.lost = True - - def logPrefix(self): - return '' - - -class TwistedIOLoop(salt.ext.tornado.ioloop.IOLoop): - """IOLoop implementation that runs on Twisted. - - `TwistedIOLoop` implements the Tornado IOLoop interface on top of - the Twisted reactor. Recommended usage:: - - from salt.ext.tornado.platform.twisted import TwistedIOLoop - from twisted.internet import reactor - TwistedIOLoop().install() - # Set up your tornado application as usual using `IOLoop.instance` - reactor.run() - - Uses the global Twisted reactor by default. To create multiple - ``TwistedIOLoops`` in the same process, you must pass a unique reactor - when constructing each one. - - Not compatible with `tornado.process.Subprocess.set_exit_callback` - because the ``SIGCHLD`` handlers used by Tornado and Twisted conflict - with each other. - - See also :meth:`tornado.ioloop.IOLoop.install` for general notes on - installing alternative IOLoops. - """ - def initialize(self, reactor=None, **kwargs): - super(TwistedIOLoop, self).initialize(**kwargs) - if reactor is None: - import twisted.internet.reactor # type: ignore - reactor = twisted.internet.reactor - self.reactor = reactor - self.fds = {} - - def close(self, all_fds=False): - fds = self.fds - self.reactor.removeAll() - for c in self.reactor.getDelayedCalls(): - c.cancel() - if all_fds: - for fd in fds.values(): - self.close_fd(fd.fileobj) - - def add_handler(self, fd, handler, events): - if fd in self.fds: - raise ValueError('fd %s added twice' % fd) - fd, fileobj = self.split_fd(fd) - self.fds[fd] = _FD(fd, fileobj, wrap(handler)) - if events & salt.ext.tornado.ioloop.IOLoop.READ: - self.fds[fd].reading = True - self.reactor.addReader(self.fds[fd]) - if events & salt.ext.tornado.ioloop.IOLoop.WRITE: - self.fds[fd].writing = True - self.reactor.addWriter(self.fds[fd]) - - def update_handler(self, fd, events): - fd, fileobj = self.split_fd(fd) - if events & salt.ext.tornado.ioloop.IOLoop.READ: - if not self.fds[fd].reading: - self.fds[fd].reading = True - self.reactor.addReader(self.fds[fd]) - else: - if self.fds[fd].reading: - self.fds[fd].reading = False - self.reactor.removeReader(self.fds[fd]) - if events & salt.ext.tornado.ioloop.IOLoop.WRITE: - if not self.fds[fd].writing: - self.fds[fd].writing = True - self.reactor.addWriter(self.fds[fd]) - else: - if self.fds[fd].writing: - self.fds[fd].writing = False - self.reactor.removeWriter(self.fds[fd]) - - def remove_handler(self, fd): - fd, fileobj = self.split_fd(fd) - if fd not in self.fds: - return - self.fds[fd].lost = True - if self.fds[fd].reading: - self.reactor.removeReader(self.fds[fd]) - if self.fds[fd].writing: - self.reactor.removeWriter(self.fds[fd]) - del self.fds[fd] - - def start(self): - old_current = IOLoop.current(instance=False) - try: - self._setup_logging() - self.make_current() - self.reactor.run() - finally: - if old_current is None: - IOLoop.clear_current() - else: - old_current.make_current() - - def stop(self): - self.reactor.crash() - - def add_timeout(self, deadline, callback, *args, **kwargs): - # This method could be simplified (since tornado 4.0) by - # overriding call_at instead of add_timeout, but we leave it - # for now as a test of backwards-compatibility. - if isinstance(deadline, numbers.Real): - delay = max(deadline - self.time(), 0) - elif isinstance(deadline, datetime.timedelta): - delay = timedelta_to_seconds(deadline) - else: - raise TypeError("Unsupported deadline %r") - return self.reactor.callLater( - delay, self._run_callback, - functools.partial(wrap(callback), *args, **kwargs)) - - def remove_timeout(self, timeout): - if timeout.active(): - timeout.cancel() - - def add_callback(self, callback, *args, **kwargs): - self.reactor.callFromThread( - self._run_callback, - functools.partial(wrap(callback), *args, **kwargs)) - - def add_callback_from_signal(self, callback, *args, **kwargs): - self.add_callback(callback, *args, **kwargs) - - -class TwistedResolver(Resolver): - """Twisted-based asynchronous resolver. - - This is a non-blocking and non-threaded resolver. It is - recommended only when threads cannot be used, since it has - limitations compared to the standard ``getaddrinfo``-based - `~tornado.netutil.Resolver` and - `~tornado.netutil.ThreadedResolver`. Specifically, it returns at - most one result, and arguments other than ``host`` and ``family`` - are ignored. It may fail to resolve when ``family`` is not - ``socket.AF_UNSPEC``. - - Requires Twisted 12.1 or newer. - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - """ - def initialize(self, io_loop=None): - self.io_loop = io_loop or IOLoop.current() - # partial copy of twisted.names.client.createResolver, which doesn't - # allow for a reactor to be passed in. - self.reactor = salt.ext.tornado.platform.twisted.TornadoReactor(io_loop) - - host_resolver = twisted.names.hosts.Resolver('/etc/hosts') - cache_resolver = twisted.names.cache.CacheResolver(reactor=self.reactor) - real_resolver = twisted.names.client.Resolver('/etc/resolv.conf', - reactor=self.reactor) - self.resolver = twisted.names.resolve.ResolverChain( - [host_resolver, cache_resolver, real_resolver]) - - @gen.coroutine - def resolve(self, host, port, family=0): - # getHostByName doesn't accept IP addresses, so if the input - # looks like an IP address just return it immediately. - if twisted.internet.abstract.isIPAddress(host): - resolved = host - resolved_family = socket.AF_INET - elif twisted.internet.abstract.isIPv6Address(host): - resolved = host - resolved_family = socket.AF_INET6 - else: - deferred = self.resolver.getHostByName(utf8(host)) - resolved = yield gen.Task(deferred.addBoth) - if isinstance(resolved, failure.Failure): - try: - resolved.raiseException() - except twisted.names.error.DomainError as e: - raise IOError(e) - elif twisted.internet.abstract.isIPAddress(resolved): - resolved_family = socket.AF_INET - elif twisted.internet.abstract.isIPv6Address(resolved): - resolved_family = socket.AF_INET6 - else: - resolved_family = socket.AF_UNSPEC - if family != socket.AF_UNSPEC and family != resolved_family: - raise Exception('Requested socket family %d but got %d' % - (family, resolved_family)) - result = [ - (resolved_family, (resolved, port)), - ] - raise gen.Return(result) - - -if hasattr(gen.convert_yielded, 'register'): - @gen.convert_yielded.register(Deferred) # type: ignore - def _(d): - f = Future() - - def errback(failure): - try: - failure.raiseException() - # Should never happen, but just in case - raise Exception("errback called without error") - except: - f.set_exc_info(sys.exc_info()) - d.addCallbacks(f.set_result, errback) - return f diff --git a/salt/ext/tornado/platform/windows.py b/salt/ext/tornado/platform/windows.py deleted file mode 100644 index 2edaf54430f..00000000000 --- a/salt/ext/tornado/platform/windows.py +++ /dev/null @@ -1,21 +0,0 @@ -# NOTE: win32 support is currently experimental, and not recommended -# for production use. -# pylint: skip-file - - -from __future__ import absolute_import, division, print_function -import ctypes # type: ignore -import ctypes.wintypes # type: ignore - -# See: http://msdn.microsoft.com/en-us/library/ms724935(VS.85).aspx -SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation -SetHandleInformation.argtypes = (ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD, ctypes.wintypes.DWORD) -SetHandleInformation.restype = ctypes.wintypes.BOOL - -HANDLE_FLAG_INHERIT = 0x00000001 - - -def set_close_exec(fd): - success = SetHandleInformation(fd, HANDLE_FLAG_INHERIT, 0) - if not success: - raise ctypes.WinError() diff --git a/salt/ext/tornado/process.py b/salt/ext/tornado/process.py deleted file mode 100644 index 93394897cd1..00000000000 --- a/salt/ext/tornado/process.py +++ /dev/null @@ -1,366 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2011 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Utilities for working with multiple processes, including both forking -the server into multiple processes and managing subprocesses. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import errno -import os -import signal -import subprocess -import sys -import time - -from binascii import hexlify - -from salt.ext.tornado.concurrent import Future -from salt.ext.tornado import ioloop -from salt.ext.tornado.iostream import PipeIOStream -from salt.ext.tornado.log import gen_log -from salt.ext.tornado.platform.auto import set_close_exec -from salt.ext.tornado import stack_context -from salt.ext.tornado.util import errno_from_exception, PY3 - -try: - import multiprocessing -except ImportError: - # Multiprocessing is not available on Google App Engine. - multiprocessing = None - -if PY3: - long = int - -# Re-export this exception for convenience. -try: - CalledProcessError = subprocess.CalledProcessError -except AttributeError: - # The subprocess module exists in Google App Engine, but is empty. - # This module isn't very useful in that case, but it should - # at least be importable. - if 'APPENGINE_RUNTIME' not in os.environ: - raise - - -def cpu_count(): - """Returns the number of processors on this machine.""" - if multiprocessing is None: - return 1 - try: - return multiprocessing.cpu_count() - except NotImplementedError: - pass - try: - return os.sysconf("SC_NPROCESSORS_CONF") - except (AttributeError, ValueError): - pass - gen_log.error("Could not detect number of processors; assuming 1") - return 1 - - -def _reseed_random(): - if 'random' not in sys.modules: - return - import random - # If os.urandom is available, this method does the same thing as - # random.seed (at least as of python 2.6). If os.urandom is not - # available, we mix in the pid in addition to a timestamp. - try: - seed = long(hexlify(os.urandom(16)), 16) - except NotImplementedError: - seed = int(time.time() * 1000) ^ os.getpid() - random.seed(seed) - - -def _pipe_cloexec(): - r, w = os.pipe() - set_close_exec(r) - set_close_exec(w) - return r, w - - -_task_id = None - - -def fork_processes(num_processes, max_restarts=100): - """Starts multiple worker processes. - - If ``num_processes`` is None or <= 0, we detect the number of cores - available on this machine and fork that number of child - processes. If ``num_processes`` is given and > 0, we fork that - specific number of sub-processes. - - Since we use processes and not threads, there is no shared memory - between any server code. - - Note that multiple processes are not compatible with the autoreload - module (or the ``autoreload=True`` option to `tornado.web.Application` - which defaults to True when ``debug=True``). - When using multiple processes, no IOLoops can be created or - referenced until after the call to ``fork_processes``. - - In each child process, ``fork_processes`` returns its *task id*, a - number between 0 and ``num_processes``. Processes that exit - abnormally (due to a signal or non-zero exit status) are restarted - with the same id (up to ``max_restarts`` times). In the parent - process, ``fork_processes`` returns None if all child processes - have exited normally, but will otherwise only exit by throwing an - exception. - """ - global _task_id - assert _task_id is None - if num_processes is None or num_processes <= 0: - num_processes = cpu_count() - if ioloop.IOLoop.initialized(): - raise RuntimeError("Cannot run in multiple processes: IOLoop instance " - "has already been initialized. You cannot call " - "IOLoop.instance() before calling start_processes()") - gen_log.info("Starting %d processes", num_processes) - children = {} - - def start_child(i): - pid = os.fork() - if pid == 0: - # child process - _reseed_random() - global _task_id - _task_id = i - return i - else: - children[pid] = i - return None - - for i in range(num_processes): - id = start_child(i) - if id is not None: - return id - num_restarts = 0 - while children: - try: - pid, status = os.wait() - except OSError as e: - if errno_from_exception(e) == errno.EINTR: - continue - raise - if pid not in children: - continue - id = children.pop(pid) - if os.WIFSIGNALED(status): - gen_log.warning("child %d (pid %d) killed by signal %d, restarting", - id, pid, os.WTERMSIG(status)) - elif os.WEXITSTATUS(status) != 0: - gen_log.warning("child %d (pid %d) exited with status %d, restarting", - id, pid, os.WEXITSTATUS(status)) - else: - gen_log.info("child %d (pid %d) exited normally", id, pid) - continue - num_restarts += 1 - if num_restarts > max_restarts: - raise RuntimeError("Too many child restarts, giving up") - new_id = start_child(id) - if new_id is not None: - return new_id - # All child processes exited cleanly, so exit the master process - # instead of just returning to right after the call to - # fork_processes (which will probably just start up another IOLoop - # unless the caller checks the return value). - sys.exit(0) - - -def task_id(): - """Returns the current task id, if any. - - Returns None if this process was not created by `fork_processes`. - """ - global _task_id - return _task_id - - -class Subprocess(object): - """Wraps ``subprocess.Popen`` with IOStream support. - - The constructor is the same as ``subprocess.Popen`` with the following - additions: - - * ``stdin``, ``stdout``, and ``stderr`` may have the value - ``tornado.process.Subprocess.STREAM``, which will make the corresponding - attribute of the resulting Subprocess a `.PipeIOStream`. - * A new keyword argument ``io_loop`` may be used to pass in an IOLoop. - - The ``Subprocess.STREAM`` option and the ``set_exit_callback`` and - ``wait_for_exit`` methods do not work on Windows. There is - therefore no reason to use this class instead of - ``subprocess.Popen`` on that platform. - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - - """ - STREAM = object() - - _initialized = False - _waiting = {} # type: ignore - - def __init__(self, *args, **kwargs): - self.io_loop = kwargs.pop('io_loop', None) or ioloop.IOLoop.current() - # All FDs we create should be closed on error; those in to_close - # should be closed in the parent process on success. - pipe_fds = [] - to_close = [] - if kwargs.get('stdin') is Subprocess.STREAM: - in_r, in_w = _pipe_cloexec() - kwargs['stdin'] = in_r - pipe_fds.extend((in_r, in_w)) - to_close.append(in_r) - self.stdin = PipeIOStream(in_w, io_loop=self.io_loop) - if kwargs.get('stdout') is Subprocess.STREAM: - out_r, out_w = _pipe_cloexec() - kwargs['stdout'] = out_w - pipe_fds.extend((out_r, out_w)) - to_close.append(out_w) - self.stdout = PipeIOStream(out_r, io_loop=self.io_loop) - if kwargs.get('stderr') is Subprocess.STREAM: - err_r, err_w = _pipe_cloexec() - kwargs['stderr'] = err_w - pipe_fds.extend((err_r, err_w)) - to_close.append(err_w) - self.stderr = PipeIOStream(err_r, io_loop=self.io_loop) - try: - self.proc = subprocess.Popen(*args, **kwargs) - except: - for fd in pipe_fds: - os.close(fd) - raise - for fd in to_close: - os.close(fd) - for attr in ['stdin', 'stdout', 'stderr', 'pid']: - if not hasattr(self, attr): # don't clobber streams set above - setattr(self, attr, getattr(self.proc, attr)) - self._exit_callback = None - self.returncode = None - - def set_exit_callback(self, callback): - """Runs ``callback`` when this process exits. - - The callback takes one argument, the return code of the process. - - This method uses a ``SIGCHLD`` handler, which is a global setting - and may conflict if you have other libraries trying to handle the - same signal. If you are using more than one ``IOLoop`` it may - be necessary to call `Subprocess.initialize` first to designate - one ``IOLoop`` to run the signal handlers. - - In many cases a close callback on the stdout or stderr streams - can be used as an alternative to an exit callback if the - signal handler is causing a problem. - """ - self._exit_callback = stack_context.wrap(callback) - Subprocess.initialize(self.io_loop) - Subprocess._waiting[self.pid] = self - Subprocess._try_cleanup_process(self.pid) - - def wait_for_exit(self, raise_error=True): - """Returns a `.Future` which resolves when the process exits. - - Usage:: - - ret = yield proc.wait_for_exit() - - This is a coroutine-friendly alternative to `set_exit_callback` - (and a replacement for the blocking `subprocess.Popen.wait`). - - By default, raises `subprocess.CalledProcessError` if the process - has a non-zero exit status. Use ``wait_for_exit(raise_error=False)`` - to suppress this behavior and return the exit status without raising. - - .. versionadded:: 4.2 - """ - future = Future() - - def callback(ret): - if ret != 0 and raise_error: - # Unfortunately we don't have the original args any more. - future.set_exception(CalledProcessError(ret, None)) - else: - future.set_result(ret) - self.set_exit_callback(callback) - return future - - @classmethod - def initialize(cls, io_loop=None): - """Initializes the ``SIGCHLD`` handler. - - The signal handler is run on an `.IOLoop` to avoid locking issues. - Note that the `.IOLoop` used for signal handling need not be the - same one used by individual Subprocess objects (as long as the - ``IOLoops`` are each running in separate threads). - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - """ - if cls._initialized: - return - if io_loop is None: - io_loop = ioloop.IOLoop.current() - cls._old_sigchld = signal.signal( - signal.SIGCHLD, - lambda sig, frame: io_loop.add_callback_from_signal(cls._cleanup)) - cls._initialized = True - - @classmethod - def uninitialize(cls): - """Removes the ``SIGCHLD`` handler.""" - if not cls._initialized: - return - signal.signal(signal.SIGCHLD, cls._old_sigchld) - cls._initialized = False - - @classmethod - def _cleanup(cls): - for pid in list(cls._waiting.keys()): # make a copy - cls._try_cleanup_process(pid) - - @classmethod - def _try_cleanup_process(cls, pid): - try: - ret_pid, status = os.waitpid(pid, os.WNOHANG) - except OSError as e: - if errno_from_exception(e) == errno.ECHILD: - return - if ret_pid == 0: - return - assert ret_pid == pid - subproc = cls._waiting.pop(pid) - subproc.io_loop.add_callback_from_signal( - subproc._set_returncode, status) - - def _set_returncode(self, status): - if os.WIFSIGNALED(status): - self.returncode = -os.WTERMSIG(status) - else: - assert os.WIFEXITED(status) - self.returncode = os.WEXITSTATUS(status) - # We've taken over wait() duty from the subprocess.Popen - # object. If we don't inform it of the process's return code, - # it will log a warning at destruction in python 3.6+. - self.proc.returncode = self.returncode - if self._exit_callback: - callback = self._exit_callback - self._exit_callback = None - callback(self.returncode) diff --git a/salt/ext/tornado/queues.py b/salt/ext/tornado/queues.py deleted file mode 100644 index 0aa146490ae..00000000000 --- a/salt/ext/tornado/queues.py +++ /dev/null @@ -1,367 +0,0 @@ -# Copyright 2015 The Tornado Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Asynchronous queues for coroutines. - -.. warning:: - - Unlike the standard library's `queue` module, the classes defined here - are *not* thread-safe. To use these queues from another thread, - use `.IOLoop.add_callback` to transfer control to the `.IOLoop` thread - before calling any queue methods. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import collections -import heapq - -from salt.ext.tornado import gen, ioloop -from salt.ext.tornado.concurrent import Future -from salt.ext.tornado.locks import Event - -__all__ = ['Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty'] - - -class QueueEmpty(Exception): - """Raised by `.Queue.get_nowait` when the queue has no items.""" - pass - - -class QueueFull(Exception): - """Raised by `.Queue.put_nowait` when a queue is at its maximum size.""" - pass - - -def _set_timeout(future, timeout): - if timeout: - def on_timeout(): - future.set_exception(gen.TimeoutError()) - io_loop = ioloop.IOLoop.current() - timeout_handle = io_loop.add_timeout(timeout, on_timeout) - future.add_done_callback( - lambda _: io_loop.remove_timeout(timeout_handle)) - - -class _QueueIterator(object): - def __init__(self, q): - self.q = q - - def __anext__(self): - return self.q.get() - - -class Queue(object): - """Coordinate producer and consumer coroutines. - - If maxsize is 0 (the default) the queue size is unbounded. - - .. testcode:: - - from salt.ext.tornado import gen - from salt.ext.tornado.ioloop import IOLoop - from salt.ext.tornado.queues import Queue - - q = Queue(maxsize=2) - - @gen.coroutine - def consumer(): - while True: - item = yield q.get() - try: - print('Doing work on %s' % item) - yield gen.sleep(0.01) - finally: - q.task_done() - - @gen.coroutine - def producer(): - for item in range(5): - yield q.put(item) - print('Put %s' % item) - - @gen.coroutine - def main(): - # Start consumer without waiting (since it never finishes). - IOLoop.current().spawn_callback(consumer) - yield producer() # Wait for producer to put all tasks. - yield q.join() # Wait for consumer to finish all tasks. - print('Done') - - IOLoop.current().run_sync(main) - - .. testoutput:: - - Put 0 - Put 1 - Doing work on 0 - Put 2 - Doing work on 1 - Put 3 - Doing work on 2 - Put 4 - Doing work on 3 - Doing work on 4 - Done - - In Python 3.5, `Queue` implements the async iterator protocol, so - ``consumer()`` could be rewritten as:: - - async def consumer(): - async for item in q: - try: - print('Doing work on %s' % item) - yield gen.sleep(0.01) - finally: - q.task_done() - - .. versionchanged:: 4.3 - Added ``async for`` support in Python 3.5. - - """ - def __init__(self, maxsize=0): - if maxsize is None: - raise TypeError("maxsize can't be None") - - if maxsize < 0: - raise ValueError("maxsize can't be negative") - - self._maxsize = maxsize - self._init() - self._getters = collections.deque([]) # Futures. - self._putters = collections.deque([]) # Pairs of (item, Future). - self._unfinished_tasks = 0 - self._finished = Event() - self._finished.set() - - @property - def maxsize(self): - """Number of items allowed in the queue.""" - return self._maxsize - - def qsize(self): - """Number of items in the queue.""" - return len(self._queue) - - def empty(self): - return not self._queue - - def full(self): - if self.maxsize == 0: - return False - else: - return self.qsize() >= self.maxsize - - def put(self, item, timeout=None): - """Put an item into the queue, perhaps waiting until there is room. - - Returns a Future, which raises `tornado.gen.TimeoutError` after a - timeout. - """ - try: - self.put_nowait(item) - except QueueFull: - future = Future() - self._putters.append((item, future)) - _set_timeout(future, timeout) - return future - else: - return gen._null_future - - def put_nowait(self, item): - """Put an item into the queue without blocking. - - If no free slot is immediately available, raise `QueueFull`. - """ - self._consume_expired() - if self._getters: - assert self.empty(), "queue non-empty, why are getters waiting?" - getter = self._getters.popleft() - self.__put_internal(item) - getter.set_result(self._get()) - elif self.full(): - raise QueueFull - else: - self.__put_internal(item) - - def get(self, timeout=None): - """Remove and return an item from the queue. - - Returns a Future which resolves once an item is available, or raises - `tornado.gen.TimeoutError` after a timeout. - """ - future = Future() - try: - future.set_result(self.get_nowait()) - except QueueEmpty: - self._getters.append(future) - _set_timeout(future, timeout) - return future - - def get_nowait(self): - """Remove and return an item from the queue without blocking. - - Return an item if one is immediately available, else raise - `QueueEmpty`. - """ - self._consume_expired() - if self._putters: - assert self.full(), "queue not full, why are putters waiting?" - item, putter = self._putters.popleft() - self.__put_internal(item) - putter.set_result(None) - return self._get() - elif self.qsize(): - return self._get() - else: - raise QueueEmpty - - def task_done(self): - """Indicate that a formerly enqueued task is complete. - - Used by queue consumers. For each `.get` used to fetch a task, a - subsequent call to `.task_done` tells the queue that the processing - on the task is complete. - - If a `.join` is blocking, it resumes when all items have been - processed; that is, when every `.put` is matched by a `.task_done`. - - Raises `ValueError` if called more times than `.put`. - """ - if self._unfinished_tasks <= 0: - raise ValueError('task_done() called too many times') - self._unfinished_tasks -= 1 - if self._unfinished_tasks == 0: - self._finished.set() - - def join(self, timeout=None): - """Block until all items in the queue are processed. - - Returns a Future, which raises `tornado.gen.TimeoutError` after a - timeout. - """ - return self._finished.wait(timeout) - - def __aiter__(self): - return _QueueIterator(self) - - # These three are overridable in subclasses. - def _init(self): - self._queue = collections.deque() - - def _get(self): - return self._queue.popleft() - - def _put(self, item): - self._queue.append(item) - # End of the overridable methods. - - def __put_internal(self, item): - self._unfinished_tasks += 1 - self._finished.clear() - self._put(item) - - def _consume_expired(self): - # Remove timed-out waiters. - while self._putters and self._putters[0][1].done(): - self._putters.popleft() - - while self._getters and self._getters[0].done(): - self._getters.popleft() - - def __repr__(self): - return '<%s at %s %s>' % ( - type(self).__name__, hex(id(self)), self._format()) - - def __str__(self): - return '<%s %s>' % (type(self).__name__, self._format()) - - def _format(self): - result = 'maxsize=%r' % (self.maxsize, ) - if getattr(self, '_queue', None): - result += ' queue=%r' % self._queue - if self._getters: - result += ' getters[%s]' % len(self._getters) - if self._putters: - result += ' putters[%s]' % len(self._putters) - if self._unfinished_tasks: - result += ' tasks=%s' % self._unfinished_tasks - return result - - -class PriorityQueue(Queue): - """A `.Queue` that retrieves entries in priority order, lowest first. - - Entries are typically tuples like ``(priority number, data)``. - - .. testcode:: - - from salt.ext.tornado.queues import PriorityQueue - - q = PriorityQueue() - q.put((1, 'medium-priority item')) - q.put((0, 'high-priority item')) - q.put((10, 'low-priority item')) - - print(q.get_nowait()) - print(q.get_nowait()) - print(q.get_nowait()) - - .. testoutput:: - - (0, 'high-priority item') - (1, 'medium-priority item') - (10, 'low-priority item') - """ - def _init(self): - self._queue = [] - - def _put(self, item): - heapq.heappush(self._queue, item) - - def _get(self): - return heapq.heappop(self._queue) - - -class LifoQueue(Queue): - """A `.Queue` that retrieves the most recently put items first. - - .. testcode:: - - from salt.ext.tornado.queues import LifoQueue - - q = LifoQueue() - q.put(3) - q.put(2) - q.put(1) - - print(q.get_nowait()) - print(q.get_nowait()) - print(q.get_nowait()) - - .. testoutput:: - - 1 - 2 - 3 - """ - def _init(self): - self._queue = [] - - def _put(self, item): - self._queue.append(item) - - def _get(self): - return self._queue.pop() diff --git a/salt/ext/tornado/routing.py b/salt/ext/tornado/routing.py deleted file mode 100644 index eba6b5d0d34..00000000000 --- a/salt/ext/tornado/routing.py +++ /dev/null @@ -1,626 +0,0 @@ -# Copyright 2015 The Tornado Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Flexible routing implementation. - -Tornado routes HTTP requests to appropriate handlers using `Router` -class implementations. The `tornado.web.Application` class is a -`Router` implementation and may be used directly, or the classes in -this module may be used for additional flexibility. The `RuleRouter` -class can match on more criteria than `.Application`, or the `Router` -interface can be subclassed for maximum customization. - -`Router` interface extends `~.httputil.HTTPServerConnectionDelegate` -to provide additional routing capabilities. This also means that any -`Router` implementation can be used directly as a ``request_callback`` -for `~.httpserver.HTTPServer` constructor. - -`Router` subclass must implement a ``find_handler`` method to provide -a suitable `~.httputil.HTTPMessageDelegate` instance to handle the -request: - -.. code-block:: python - - class CustomRouter(Router): - def find_handler(self, request, **kwargs): - # some routing logic providing a suitable HTTPMessageDelegate instance - return MessageDelegate(request.connection) - - class MessageDelegate(HTTPMessageDelegate): - def __init__(self, connection): - self.connection = connection - - def finish(self): - self.connection.write_headers( - ResponseStartLine("HTTP/1.1", 200, "OK"), - HTTPHeaders({"Content-Length": "2"}), - b"OK") - self.connection.finish() - - router = CustomRouter() - server = HTTPServer(router) - -The main responsibility of `Router` implementation is to provide a -mapping from a request to `~.httputil.HTTPMessageDelegate` instance -that will handle this request. In the example above we can see that -routing is possible even without instantiating an `~.web.Application`. - -For routing to `~.web.RequestHandler` implementations we need an -`~.web.Application` instance. `~.web.Application.get_handler_delegate` -provides a convenient way to create `~.httputil.HTTPMessageDelegate` -for a given request and `~.web.RequestHandler`. - -Here is a simple example of how we can we route to -`~.web.RequestHandler` subclasses by HTTP method: - -.. code-block:: python - - resources = {} - - class GetResource(RequestHandler): - def get(self, path): - if path not in resources: - raise HTTPError(404) - - self.finish(resources[path]) - - class PostResource(RequestHandler): - def post(self, path): - resources[path] = self.request.body - - class HTTPMethodRouter(Router): - def __init__(self, app): - self.app = app - - def find_handler(self, request, **kwargs): - handler = GetResource if request.method == "GET" else PostResource - return self.app.get_handler_delegate(request, handler, path_args=[request.path]) - - router = HTTPMethodRouter(Application()) - server = HTTPServer(router) - -`ReversibleRouter` interface adds the ability to distinguish between -the routes and reverse them to the original urls using route's name -and additional arguments. `~.web.Application` is itself an -implementation of `ReversibleRouter` class. - -`RuleRouter` and `ReversibleRuleRouter` are implementations of -`Router` and `ReversibleRouter` interfaces and can be used for -creating rule-based routing configurations. - -Rules are instances of `Rule` class. They contain a `Matcher`, which -provides the logic for determining whether the rule is a match for a -particular request and a target, which can be one of the following. - -1) An instance of `~.httputil.HTTPServerConnectionDelegate`: - -.. code-block:: python - - router = RuleRouter([ - Rule(PathMatches("/handler"), ConnectionDelegate()), - # ... more rules - ]) - - class ConnectionDelegate(HTTPServerConnectionDelegate): - def start_request(self, server_conn, request_conn): - return MessageDelegate(request_conn) - -2) A callable accepting a single argument of `~.httputil.HTTPServerRequest` type: - -.. code-block:: python - - router = RuleRouter([ - Rule(PathMatches("/callable"), request_callable) - ]) - - def request_callable(request): - request.write(b"HTTP/1.1 200 OK\\r\\nContent-Length: 2\\r\\n\\r\\nOK") - request.finish() - -3) Another `Router` instance: - -.. code-block:: python - - router = RuleRouter([ - Rule(PathMatches("/router.*"), CustomRouter()) - ]) - -Of course a nested `RuleRouter` or a `~.web.Application` is allowed: - -.. code-block:: python - - router = RuleRouter([ - Rule(HostMatches("example.com"), RuleRouter([ - Rule(PathMatches("/app1/.*"), Application([(r"/app1/handler", Handler)]))), - ])) - ]) - - server = HTTPServer(router) - -In the example below `RuleRouter` is used to route between applications: - -.. code-block:: python - - app1 = Application([ - (r"/app1/handler", Handler1), - # other handlers ... - ]) - - app2 = Application([ - (r"/app2/handler", Handler2), - # other handlers ... - ]) - - router = RuleRouter([ - Rule(PathMatches("/app1.*"), app1), - Rule(PathMatches("/app2.*"), app2) - ]) - - server = HTTPServer(router) - -For more information on application-level routing see docs for `~.web.Application`. - -.. versionadded:: 4.5 - -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import re -from functools import partial - -from salt.ext.tornado import httputil -from salt.ext.tornado.httpserver import _CallableAdapter -from salt.ext.tornado.escape import url_escape, url_unescape, utf8 -from salt.ext.tornado.log import app_log -from salt.ext.tornado.util import basestring_type, import_object, re_unescape, unicode_type - -try: - import typing # noqa -except ImportError: - pass - - -class Router(httputil.HTTPServerConnectionDelegate): - """Abstract router interface.""" - - def find_handler(self, request, **kwargs): - # type: (httputil.HTTPServerRequest, typing.Any)->httputil.HTTPMessageDelegate - """Must be implemented to return an appropriate instance of `~.httputil.HTTPMessageDelegate` - that can serve the request. - Routing implementations may pass additional kwargs to extend the routing logic. - - :arg httputil.HTTPServerRequest request: current HTTP request. - :arg kwargs: additional keyword arguments passed by routing implementation. - :returns: an instance of `~.httputil.HTTPMessageDelegate` that will be used to - process the request. - """ - raise NotImplementedError() - - def start_request(self, server_conn, request_conn): - return _RoutingDelegate(self, server_conn, request_conn) - - -class ReversibleRouter(Router): - """Abstract router interface for routers that can handle named routes - and support reversing them to original urls. - """ - - def reverse_url(self, name, *args): - """Returns url string for a given route name and arguments - or ``None`` if no match is found. - - :arg str name: route name. - :arg args: url parameters. - :returns: parametrized url string for a given route name (or ``None``). - """ - raise NotImplementedError() - - -class _RoutingDelegate(httputil.HTTPMessageDelegate): - def __init__(self, router, server_conn, request_conn): - self.server_conn = server_conn - self.request_conn = request_conn - self.delegate = None - self.router = router # type: Router - - def headers_received(self, start_line, headers): - request = httputil.HTTPServerRequest( - connection=self.request_conn, - server_connection=self.server_conn, - start_line=start_line, headers=headers) - - self.delegate = self.router.find_handler(request) - return self.delegate.headers_received(start_line, headers) - - def data_received(self, chunk): - return self.delegate.data_received(chunk) - - def finish(self): - self.delegate.finish() - - def on_connection_close(self): - self.delegate.on_connection_close() - - -class RuleRouter(Router): - """Rule-based router implementation.""" - - def __init__(self, rules=None): - """Constructs a router from an ordered list of rules:: - - RuleRouter([ - Rule(PathMatches("/handler"), Target), - # ... more rules - ]) - - You can also omit explicit `Rule` constructor and use tuples of arguments:: - - RuleRouter([ - (PathMatches("/handler"), Target), - ]) - - `PathMatches` is a default matcher, so the example above can be simplified:: - - RuleRouter([ - ("/handler", Target), - ]) - - In the examples above, ``Target`` can be a nested `Router` instance, an instance of - `~.httputil.HTTPServerConnectionDelegate` or an old-style callable, accepting a request argument. - - :arg rules: a list of `Rule` instances or tuples of `Rule` - constructor arguments. - """ - self.rules = [] # type: typing.List[Rule] - if rules: - self.add_rules(rules) - - def add_rules(self, rules): - """Appends new rules to the router. - - :arg rules: a list of Rule instances (or tuples of arguments, which are - passed to Rule constructor). - """ - for rule in rules: - if isinstance(rule, (tuple, list)): - assert len(rule) in (2, 3, 4) - if isinstance(rule[0], basestring_type): - rule = Rule(PathMatches(rule[0]), *rule[1:]) - else: - rule = Rule(*rule) - - self.rules.append(self.process_rule(rule)) - - def process_rule(self, rule): - """Override this method for additional preprocessing of each rule. - - :arg Rule rule: a rule to be processed. - :returns: the same or modified Rule instance. - """ - return rule - - def find_handler(self, request, **kwargs): - for rule in self.rules: - target_params = rule.matcher.match(request) - if target_params is not None: - if rule.target_kwargs: - target_params['target_kwargs'] = rule.target_kwargs - - delegate = self.get_target_delegate( - rule.target, request, **target_params) - - if delegate is not None: - return delegate - - return None - - def get_target_delegate(self, target, request, **target_params): - """Returns an instance of `~.httputil.HTTPMessageDelegate` for a - Rule's target. This method is called by `~.find_handler` and can be - extended to provide additional target types. - - :arg target: a Rule's target. - :arg httputil.HTTPServerRequest request: current request. - :arg target_params: additional parameters that can be useful - for `~.httputil.HTTPMessageDelegate` creation. - """ - if isinstance(target, Router): - return target.find_handler(request, **target_params) - - elif isinstance(target, httputil.HTTPServerConnectionDelegate): - return target.start_request(request.server_connection, request.connection) - - elif callable(target): - return _CallableAdapter( - partial(target, **target_params), request.connection - ) - - return None - - -class ReversibleRuleRouter(ReversibleRouter, RuleRouter): - """A rule-based router that implements ``reverse_url`` method. - - Each rule added to this router may have a ``name`` attribute that can be - used to reconstruct an original uri. The actual reconstruction takes place - in a rule's matcher (see `Matcher.reverse`). - """ - - def __init__(self, rules=None): - self.named_rules = {} # type: typing.Dict[str] - super(ReversibleRuleRouter, self).__init__(rules) - - def process_rule(self, rule): - rule = super(ReversibleRuleRouter, self).process_rule(rule) - - if rule.name: - if rule.name in self.named_rules: - app_log.warning( - "Multiple handlers named %s; replacing previous value", - rule.name) - self.named_rules[rule.name] = rule - - return rule - - def reverse_url(self, name, *args): - if name in self.named_rules: - return self.named_rules[name].matcher.reverse(*args) - - for rule in self.rules: - if isinstance(rule.target, ReversibleRouter): - reversed_url = rule.target.reverse_url(name, *args) - if reversed_url is not None: - return reversed_url - - return None - - -class Rule(object): - """A routing rule.""" - - def __init__(self, matcher, target, target_kwargs=None, name=None): - """Constructs a Rule instance. - - :arg Matcher matcher: a `Matcher` instance used for determining - whether the rule should be considered a match for a specific - request. - :arg target: a Rule's target (typically a ``RequestHandler`` or - `~.httputil.HTTPServerConnectionDelegate` subclass or even a nested `Router`, - depending on routing implementation). - :arg dict target_kwargs: a dict of parameters that can be useful - at the moment of target instantiation (for example, ``status_code`` - for a ``RequestHandler`` subclass). They end up in - ``target_params['target_kwargs']`` of `RuleRouter.get_target_delegate` - method. - :arg str name: the name of the rule that can be used to find it - in `ReversibleRouter.reverse_url` implementation. - """ - if isinstance(target, str): - # import the Module and instantiate the class - # Must be a fully qualified name (module.ClassName) - target = import_object(target) - - self.matcher = matcher # type: Matcher - self.target = target - self.target_kwargs = target_kwargs if target_kwargs else {} - self.name = name - - def reverse(self, *args): - return self.matcher.reverse(*args) - - def __repr__(self): - return '%s(%r, %s, kwargs=%r, name=%r)' % \ - (self.__class__.__name__, self.matcher, - self.target, self.target_kwargs, self.name) - - -class Matcher(object): - """Represents a matcher for request features.""" - - def match(self, request): - """Matches current instance against the request. - - :arg httputil.HTTPServerRequest request: current HTTP request - :returns: a dict of parameters to be passed to the target handler - (for example, ``handler_kwargs``, ``path_args``, ``path_kwargs`` - can be passed for proper `~.web.RequestHandler` instantiation). - An empty dict is a valid (and common) return value to indicate a match - when the argument-passing features are not used. - ``None`` must be returned to indicate that there is no match.""" - raise NotImplementedError() - - def reverse(self, *args): - """Reconstructs full url from matcher instance and additional arguments.""" - return None - - -class AnyMatches(Matcher): - """Matches any request.""" - - def match(self, request): - return {} - - -class HostMatches(Matcher): - """Matches requests from hosts specified by ``host_pattern`` regex.""" - - def __init__(self, host_pattern): - if isinstance(host_pattern, basestring_type): - if not host_pattern.endswith("$"): - host_pattern += "$" - self.host_pattern = re.compile(host_pattern) - else: - self.host_pattern = host_pattern - - def match(self, request): - if self.host_pattern.match(request.host_name): - return {} - - return None - - -class DefaultHostMatches(Matcher): - """Matches requests from host that is equal to application's default_host. - Always returns no match if ``X-Real-Ip`` header is present. - """ - - def __init__(self, application, host_pattern): - self.application = application - self.host_pattern = host_pattern - - def match(self, request): - # Look for default host if not behind load balancer (for debugging) - if "X-Real-Ip" not in request.headers: - if self.host_pattern.match(self.application.default_host): - return {} - return None - - -class PathMatches(Matcher): - """Matches requests with paths specified by ``path_pattern`` regex.""" - - def __init__(self, path_pattern): - if isinstance(path_pattern, basestring_type): - if not path_pattern.endswith('$'): - path_pattern += '$' - self.regex = re.compile(path_pattern) - else: - self.regex = path_pattern - - assert len(self.regex.groupindex) in (0, self.regex.groups), \ - ("groups in url regexes must either be all named or all " - "positional: %r" % self.regex.pattern) - - self._path, self._group_count = self._find_groups() - - def match(self, request): - match = self.regex.match(request.path) - if match is None: - return None - if not self.regex.groups: - return {} - - path_args, path_kwargs = [], {} - - # Pass matched groups to the handler. Since - # match.groups() includes both named and - # unnamed groups, we want to use either groups - # or groupdict but not both. - if self.regex.groupindex: - path_kwargs = dict( - (str(k), _unquote_or_none(v)) - for (k, v) in match.groupdict().items()) - else: - path_args = [_unquote_or_none(s) for s in match.groups()] - - return dict(path_args=path_args, path_kwargs=path_kwargs) - - def reverse(self, *args): - if self._path is None: - raise ValueError("Cannot reverse url regex " + self.regex.pattern) - assert len(args) == self._group_count, "required number of arguments " \ - "not found" - if not len(args): - return self._path - converted_args = [] - for a in args: - if not isinstance(a, (unicode_type, bytes)): - a = str(a) - converted_args.append(url_escape(utf8(a), plus=False)) - return self._path % tuple(converted_args) - - def _find_groups(self): - """Returns a tuple (reverse string, group count) for a url. - - For example: Given the url pattern /([0-9]{4})/([a-z-]+)/, this method - would return ('/%s/%s/', 2). - """ - pattern = self.regex.pattern - if pattern.startswith('^'): - pattern = pattern[1:] - if pattern.endswith('$'): - pattern = pattern[:-1] - - if self.regex.groups != pattern.count('('): - # The pattern is too complicated for our simplistic matching, - # so we can't support reversing it. - return None, None - - pieces = [] - for fragment in pattern.split('('): - if ')' in fragment: - paren_loc = fragment.index(')') - if paren_loc >= 0: - pieces.append('%s' + fragment[paren_loc + 1:]) - else: - try: - unescaped_fragment = re_unescape(fragment) - except ValueError as exc: - # If we can't unescape part of it, we can't - # reverse this url. - return (None, None) - pieces.append(unescaped_fragment) - - return ''.join(pieces), self.regex.groups - - -class URLSpec(Rule): - """Specifies mappings between URLs and handlers. - - .. versionchanged: 4.5 - `URLSpec` is now a subclass of a `Rule` with `PathMatches` matcher and is preserved for - backwards compatibility. - """ - def __init__(self, pattern, handler, kwargs=None, name=None): - """Parameters: - - * ``pattern``: Regular expression to be matched. Any capturing - groups in the regex will be passed in to the handler's - get/post/etc methods as arguments (by keyword if named, by - position if unnamed. Named and unnamed capturing groups may - may not be mixed in the same rule). - - * ``handler``: `~.web.RequestHandler` subclass to be invoked. - - * ``kwargs`` (optional): A dictionary of additional arguments - to be passed to the handler's constructor. - - * ``name`` (optional): A name for this handler. Used by - `~.web.Application.reverse_url`. - - """ - super(URLSpec, self).__init__(PathMatches(pattern), handler, kwargs, name) - - self.regex = self.matcher.regex - self.handler_class = self.target - self.kwargs = kwargs - - def __repr__(self): - return '%s(%r, %s, kwargs=%r, name=%r)' % \ - (self.__class__.__name__, self.regex.pattern, - self.handler_class, self.kwargs, self.name) - - -def _unquote_or_none(s): - """None-safe wrapper around url_unescape to handle unmatched optional - groups correctly. - - Note that args are passed as bytes so the handler can decide what - encoding to use. - """ - if s is None: - return s - return url_unescape(s, encoding=None, plus=False) diff --git a/salt/ext/tornado/simple_httpclient.py b/salt/ext/tornado/simple_httpclient.py deleted file mode 100644 index 8938fe14edb..00000000000 --- a/salt/ext/tornado/simple_httpclient.py +++ /dev/null @@ -1,568 +0,0 @@ -#!/usr/bin/env python -# pylint: skip-file -from __future__ import absolute_import, division, print_function - -from salt.ext.tornado.escape import utf8, _unicode -from salt.ext.tornado import gen -from salt.ext.tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy -from salt.ext.tornado import httputil -from salt.ext.tornado.http1connection import HTTP1Connection, HTTP1ConnectionParameters -from salt.ext.tornado.iostream import StreamClosedError -from salt.ext.tornado.netutil import Resolver, OverrideResolver, _client_ssl_defaults -from salt.ext.tornado.log import gen_log -from salt.ext.tornado import stack_context -from salt.ext.tornado.tcpclient import TCPClient -from salt.ext.tornado.util import PY3 - -import base64 -import collections -import copy -import functools -import re -import socket -import sys -from io import BytesIO - - -if PY3: - import urllib.parse as urlparse -else: - import urlparse - -try: - import ssl -except ImportError: - # ssl is not available on Google App Engine. - ssl = None - -try: - import certifi -except ImportError: - certifi = None - - -def _default_ca_certs(): - if certifi is None: - raise Exception("The 'certifi' package is required to use https " - "in simple_httpclient") - return certifi.where() - - -class SimpleAsyncHTTPClient(AsyncHTTPClient): - """Non-blocking HTTP client with no external dependencies. - - This class implements an HTTP 1.1 client on top of Tornado's IOStreams. - Some features found in the curl-based AsyncHTTPClient are not yet - supported. In particular, proxies are not supported, connections - are not reused, and callers cannot select the network interface to be - used. - """ - def initialize(self, io_loop, max_clients=10, - hostname_mapping=None, max_buffer_size=104857600, - resolver=None, defaults=None, max_header_size=None, - max_body_size=None): - """Creates a AsyncHTTPClient. - - Only a single AsyncHTTPClient instance exists per IOLoop - in order to provide limitations on the number of pending connections. - ``force_instance=True`` may be used to suppress this behavior. - - Note that because of this implicit reuse, unless ``force_instance`` - is used, only the first call to the constructor actually uses - its arguments. It is recommended to use the ``configure`` method - instead of the constructor to ensure that arguments take effect. - - ``max_clients`` is the number of concurrent requests that can be - in progress; when this limit is reached additional requests will be - queued. Note that time spent waiting in this queue still counts - against the ``request_timeout``. - - ``hostname_mapping`` is a dictionary mapping hostnames to IP addresses. - It can be used to make local DNS changes when modifying system-wide - settings like ``/etc/hosts`` is not possible or desirable (e.g. in - unittests). - - ``max_buffer_size`` (default 100MB) is the number of bytes - that can be read into memory at once. ``max_body_size`` - (defaults to ``max_buffer_size``) is the largest response body - that the client will accept. Without a - ``streaming_callback``, the smaller of these two limits - applies; with a ``streaming_callback`` only ``max_body_size`` - does. - - .. versionchanged:: 4.2 - Added the ``max_body_size`` argument. - """ - super(SimpleAsyncHTTPClient, self).initialize(io_loop, - defaults=defaults) - self.max_clients = max_clients - self.queue = collections.deque() - self.active = {} - self.waiting = {} - self.max_buffer_size = max_buffer_size - self.max_header_size = max_header_size - self.max_body_size = max_body_size - # TCPClient could create a Resolver for us, but we have to do it - # ourselves to support hostname_mapping. - if resolver: - self.resolver = resolver - self.own_resolver = False - else: - self.resolver = Resolver(io_loop=io_loop) - self.own_resolver = True - if hostname_mapping is not None: - self.resolver = OverrideResolver(resolver=self.resolver, - mapping=hostname_mapping) - self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop) - - def close(self): - super(SimpleAsyncHTTPClient, self).close() - if self.own_resolver: - self.resolver.close() - self.tcp_client.close() - - def fetch_impl(self, request, callback): - key = object() - self.queue.append((key, request, callback)) - if not len(self.active) < self.max_clients: - timeout_handle = self.io_loop.add_timeout( - self.io_loop.time() + min(request.connect_timeout, - request.request_timeout), - functools.partial(self._on_timeout, key, "in request queue")) - else: - timeout_handle = None - self.waiting[key] = (request, callback, timeout_handle) - self._process_queue() - if self.queue: - gen_log.debug("max_clients limit reached, request queued. " - "%d active, %d queued requests." % ( - len(self.active), len(self.queue))) - - def _process_queue(self): - with stack_context.NullContext(): - while self.queue and len(self.active) < self.max_clients: - key, request, callback = self.queue.popleft() - if key not in self.waiting: - continue - self._remove_timeout(key) - self.active[key] = (request, callback) - release_callback = functools.partial(self._release_fetch, key) - self._handle_request(request, release_callback, callback) - - def _connection_class(self): - return _HTTPConnection - - def _handle_request(self, request, release_callback, final_callback): - self._connection_class()( - self.io_loop, self, request, release_callback, - final_callback, self.max_buffer_size, self.tcp_client, - self.max_header_size, self.max_body_size) - - def _release_fetch(self, key): - del self.active[key] - self._process_queue() - - def _remove_timeout(self, key): - if key in self.waiting: - request, callback, timeout_handle = self.waiting[key] - if timeout_handle is not None: - self.io_loop.remove_timeout(timeout_handle) - del self.waiting[key] - - def _on_timeout(self, key, info=None): - """Timeout callback of request. - - Construct a timeout HTTPResponse when a timeout occurs. - - :arg object key: A simple object to mark the request. - :info string key: More detailed timeout information. - """ - request, callback, timeout_handle = self.waiting[key] - self.queue.remove((key, request, callback)) - - error_message = "Timeout {0}".format(info) if info else "Timeout" - timeout_response = HTTPResponse( - request, 599, error=HTTPError(599, error_message), - request_time=self.io_loop.time() - request.start_time) - self.io_loop.add_callback(callback, timeout_response) - del self.waiting[key] - - -class _HTTPConnection(httputil.HTTPMessageDelegate): - _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"]) - - def __init__(self, io_loop, client, request, release_callback, - final_callback, max_buffer_size, tcp_client, - max_header_size, max_body_size): - self.start_time = io_loop.time() - self.io_loop = io_loop - self.client = client - self.request = request - self.release_callback = release_callback - self.final_callback = final_callback - self.max_buffer_size = max_buffer_size - self.tcp_client = tcp_client - self.max_header_size = max_header_size - self.max_body_size = max_body_size - self.code = None - self.headers = None - self.chunks = [] - self._decompressor = None - # Timeout handle returned by IOLoop.add_timeout - self._timeout = None - self._sockaddr = None - with stack_context.ExceptionStackContext(self._handle_exception): - self.parsed = urlparse.urlsplit(_unicode(self.request.url)) - if self.parsed.scheme not in ("http", "https"): - raise ValueError("Unsupported url scheme: %s" % - self.request.url) - # urlsplit results have hostname and port results, but they - # didn't support ipv6 literals until python 2.7. - netloc = self.parsed.netloc - if "@" in netloc: - userpass, _, netloc = netloc.rpartition("@") - host, port = httputil.split_host_and_port(netloc) - if port is None: - port = 443 if self.parsed.scheme == "https" else 80 - if re.match(r'^\[.*\]$', host): - # raw ipv6 addresses in urls are enclosed in brackets - host = host[1:-1] - self.parsed_hostname = host # save final host for _on_connect - - if request.allow_ipv6 is False: - af = socket.AF_INET - else: - af = socket.AF_UNSPEC - - ssl_options = self._get_ssl_options(self.parsed.scheme) - - timeout = min(self.request.connect_timeout, self.request.request_timeout) - if timeout: - self._timeout = self.io_loop.add_timeout( - self.start_time + timeout, - stack_context.wrap(functools.partial(self._on_timeout, "while connecting"))) - self.tcp_client.connect(host, port, af=af, - ssl_options=ssl_options, - max_buffer_size=self.max_buffer_size, - callback=self._on_connect) - - def _get_ssl_options(self, scheme): - if scheme == "https": - if self.request.ssl_options is not None: - return self.request.ssl_options - # If we are using the defaults, don't construct a - # new SSLContext. - if (self.request.validate_cert and - self.request.ca_certs is None and - self.request.client_cert is None and - self.request.client_key is None): - return _client_ssl_defaults - ssl_options = {} - if self.request.validate_cert: - ssl_options["cert_reqs"] = ssl.CERT_REQUIRED - if self.request.ca_certs is not None: - ssl_options["ca_certs"] = self.request.ca_certs - elif not hasattr(ssl, 'create_default_context'): - # When create_default_context is present, - # we can omit the "ca_certs" parameter entirely, - # which avoids the dependency on "certifi" for py34. - ssl_options["ca_certs"] = _default_ca_certs() - if self.request.client_key is not None: - ssl_options["keyfile"] = self.request.client_key - if self.request.client_cert is not None: - ssl_options["certfile"] = self.request.client_cert - - # SSL interoperability is tricky. We want to disable - # SSLv2 for security reasons; it wasn't disabled by default - # until openssl 1.0. The best way to do this is to use - # the SSL_OP_NO_SSLv2, but that wasn't exposed to python - # until 3.2. Python 2.7 adds the ciphers argument, which - # can also be used to disable SSLv2. As a last resort - # on python 2.6, we set ssl_version to TLSv1. This is - # more narrow than we'd like since it also breaks - # compatibility with servers configured for SSLv3 only, - # but nearly all servers support both SSLv3 and TLSv1: - # http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html - if sys.version_info >= (2, 7): - # In addition to disabling SSLv2, we also exclude certain - # classes of insecure ciphers. - ssl_options["ciphers"] = "DEFAULT:!SSLv2:!EXPORT:!DES" - else: - # This is really only necessary for pre-1.0 versions - # of openssl, but python 2.6 doesn't expose version - # information. - ssl_options["ssl_version"] = ssl.PROTOCOL_TLSv1 - return ssl_options - return None - - def _on_timeout(self, info=None): - """Timeout callback of _HTTPConnection instance. - - Raise a timeout HTTPError when a timeout occurs. - - :info string key: More detailed timeout information. - """ - self._timeout = None - error_message = "Timeout {0}".format(info) if info else "Timeout" - if self.final_callback is not None: - raise HTTPError(599, error_message) - - def _remove_timeout(self): - if self._timeout is not None: - self.io_loop.remove_timeout(self._timeout) - self._timeout = None - - def _on_connect(self, stream): - if self.final_callback is None: - # final_callback is cleared if we've hit our timeout. - stream.close() - return - self.stream = stream - self.stream.set_close_callback(self.on_connection_close) - self._remove_timeout() - if self.final_callback is None: - return - if self.request.request_timeout: - self._timeout = self.io_loop.add_timeout( - self.start_time + self.request.request_timeout, - stack_context.wrap(functools.partial(self._on_timeout, "during request"))) - if (self.request.method not in self._SUPPORTED_METHODS and - not self.request.allow_nonstandard_methods): - raise KeyError("unknown method %s" % self.request.method) - for key in ('network_interface', - 'proxy_host', 'proxy_port', - 'proxy_username', 'proxy_password', - 'proxy_auth_mode'): - if getattr(self.request, key, None): - raise NotImplementedError('%s not supported' % key) - if "Connection" not in self.request.headers: - self.request.headers["Connection"] = "close" - if "Host" not in self.request.headers: - if '@' in self.parsed.netloc: - self.request.headers["Host"] = self.parsed.netloc.rpartition('@')[-1] - else: - self.request.headers["Host"] = self.parsed.netloc - username, password = None, None - if self.parsed.username is not None: - username, password = self.parsed.username, self.parsed.password - elif self.request.auth_username is not None: - username = self.request.auth_username - password = self.request.auth_password or '' - if username is not None: - if self.request.auth_mode not in (None, "basic"): - raise ValueError("unsupported auth_mode %s", - self.request.auth_mode) - auth = utf8(username) + b":" + utf8(password) - self.request.headers["Authorization"] = (b"Basic " + - base64.b64encode(auth)) - if self.request.user_agent: - self.request.headers["User-Agent"] = self.request.user_agent - if not self.request.allow_nonstandard_methods: - # Some HTTP methods nearly always have bodies while others - # almost never do. Fail in this case unless the user has - # opted out of sanity checks with allow_nonstandard_methods. - body_expected = self.request.method in ("POST", "PATCH", "PUT") - body_present = (self.request.body is not None or - self.request.body_producer is not None) - if ((body_expected and not body_present) or - (body_present and not body_expected)): - raise ValueError( - 'Body must %sbe None for method %s (unless ' - 'allow_nonstandard_methods is true)' % - ('not ' if body_expected else '', self.request.method)) - if self.request.expect_100_continue: - self.request.headers["Expect"] = "100-continue" - if self.request.body is not None: - # When body_producer is used the caller is responsible for - # setting Content-Length (or else chunked encoding will be used). - self.request.headers["Content-Length"] = str(len( - self.request.body)) - if (self.request.method == "POST" and - "Content-Type" not in self.request.headers): - self.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - if self.request.decompress_response: - self.request.headers["Accept-Encoding"] = "gzip" - req_path = ((self.parsed.path or '/') + - (('?' + self.parsed.query) if self.parsed.query else '')) - self.connection = self._create_connection(stream) - start_line = httputil.RequestStartLine(self.request.method, - req_path, '') - self.connection.write_headers(start_line, self.request.headers) - if self.request.expect_100_continue: - self._read_response() - else: - self._write_body(True) - - def _create_connection(self, stream): - stream.set_nodelay(True) - connection = HTTP1Connection( - stream, True, - HTTP1ConnectionParameters( - no_keep_alive=True, - max_header_size=self.max_header_size, - max_body_size=self.max_body_size, - decompress=self.request.decompress_response), - self._sockaddr) - return connection - - def _write_body(self, start_read): - if self.request.body is not None: - self.connection.write(self.request.body) - elif self.request.body_producer is not None: - fut = self.request.body_producer(self.connection.write) - if fut is not None: - fut = gen.convert_yielded(fut) - - def on_body_written(fut): - fut.result() - self.connection.finish() - if start_read: - self._read_response() - self.io_loop.add_future(fut, on_body_written) - return - self.connection.finish() - if start_read: - self._read_response() - - def _read_response(self): - # Ensure that any exception raised in read_response ends up in our - # stack context. - self.io_loop.add_future( - self.connection.read_response(self), - lambda f: f.result()) - - def _release(self): - if self.release_callback is not None: - release_callback = self.release_callback - self.release_callback = None - release_callback() - - def _run_callback(self, response): - self._release() - if self.final_callback is not None: - final_callback = self.final_callback - self.final_callback = None - self.io_loop.add_callback(final_callback, response) - - def _handle_exception(self, typ, value, tb): - if self.final_callback: - self._remove_timeout() - if isinstance(value, StreamClosedError): - if value.real_error is None: - value = HTTPError(599, "Stream closed") - else: - value = value.real_error - self._run_callback(HTTPResponse(self.request, 599, error=value, - request_time=self.io_loop.time() - self.start_time, - )) - - if hasattr(self, "stream"): - # TODO: this may cause a StreamClosedError to be raised - # by the connection's Future. Should we cancel the - # connection more gracefully? - self.stream.close() - return True - else: - # If our callback has already been called, we are probably - # catching an exception that is not caused by us but rather - # some child of our callback. Rather than drop it on the floor, - # pass it along, unless it's just the stream being closed. - return isinstance(value, StreamClosedError) - - def on_connection_close(self): - if self.final_callback is not None: - message = "Connection closed" - if self.stream.error: - raise self.stream.error - try: - raise HTTPError(599, message) - except HTTPError: - self._handle_exception(*sys.exc_info()) - - def headers_received(self, first_line, headers): - if self.request.expect_100_continue and first_line.code == 100: - self._write_body(False) - return - self.code = first_line.code - self.reason = first_line.reason - self.headers = headers - - if self._should_follow_redirect(): - return - - if self.request.header_callback is not None: - # Reassemble the start line. - self.request.header_callback('%s %s %s\r\n' % first_line) - for k, v in self.headers.get_all(): - self.request.header_callback("%s: %s\r\n" % (k, v)) - self.request.header_callback('\r\n') - - def _should_follow_redirect(self): - return (self.request.follow_redirects and - self.request.max_redirects > 0 and - self.code in (301, 302, 303, 307, 308)) - - def finish(self): - data = b''.join(self.chunks) - self._remove_timeout() - original_request = getattr(self.request, "original_request", - self.request) - if self._should_follow_redirect(): - assert isinstance(self.request, _RequestProxy) - new_request = copy.copy(self.request.request) - new_request.url = urlparse.urljoin(self.request.url, - self.headers["Location"]) - new_request.max_redirects = self.request.max_redirects - 1 - del new_request.headers["Host"] - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 - # Client SHOULD make a GET request after a 303. - # According to the spec, 302 should be followed by the same - # method as the original request, but in practice browsers - # treat 302 the same as 303, and many servers use 302 for - # compatibility with pre-HTTP/1.1 user agents which don't - # understand the 303 status. - if self.code in (302, 303): - new_request.method = "GET" - new_request.body = None - for h in ["Content-Length", "Content-Type", - "Content-Encoding", "Transfer-Encoding"]: - try: - del self.request.headers[h] - except KeyError: - pass - new_request.original_request = original_request - final_callback = self.final_callback - self.final_callback = None - self._release() - self.client.fetch(new_request, final_callback) - self._on_end_request() - return - if self.request.streaming_callback: - buffer = BytesIO() - else: - buffer = BytesIO(data) # TODO: don't require one big string? - response = HTTPResponse(original_request, - self.code, reason=getattr(self, 'reason', None), - headers=self.headers, - request_time=self.io_loop.time() - self.start_time, - buffer=buffer, - effective_url=self.request.url) - self._run_callback(response) - self._on_end_request() - - def _on_end_request(self): - self.stream.close() - - def data_received(self, chunk): - if self._should_follow_redirect(): - # We're going to follow a redirect so just discard the body. - return - if self.request.streaming_callback is not None: - self.request.streaming_callback(chunk) - else: - self.chunks.append(chunk) - - -if __name__ == "__main__": - AsyncHTTPClient.configure(SimpleAsyncHTTPClient) - main() diff --git a/salt/ext/tornado/speedups.c b/salt/ext/tornado/speedups.c deleted file mode 100644 index c59bda00922..00000000000 --- a/salt/ext/tornado/speedups.c +++ /dev/null @@ -1,52 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include - -static PyObject* websocket_mask(PyObject* self, PyObject* args) { - const char* mask; - Py_ssize_t mask_len; - const char* data; - Py_ssize_t data_len; - Py_ssize_t i; - PyObject* result; - char* buf; - - if (!PyArg_ParseTuple(args, "s#s#", &mask, &mask_len, &data, &data_len)) { - return NULL; - } - - result = PyBytes_FromStringAndSize(NULL, data_len); - if (!result) { - return NULL; - } - buf = PyBytes_AsString(result); - for (i = 0; i < data_len; i++) { - buf[i] = data[i] ^ mask[i % 4]; - } - - return result; -} - -static PyMethodDef methods[] = { - {"websocket_mask", websocket_mask, METH_VARARGS, ""}, - {NULL, NULL, 0, NULL} -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef speedupsmodule = { - PyModuleDef_HEAD_INIT, - "speedups", - NULL, - -1, - methods -}; - -PyMODINIT_FUNC -PyInit_speedups(void) { - return PyModule_Create(&speedupsmodule); -} -#else // Python 2.x -PyMODINIT_FUNC -initspeedups(void) { - Py_InitModule("tornado.speedups", methods); -} -#endif diff --git a/salt/ext/tornado/speedups.pyi b/salt/ext/tornado/speedups.pyi deleted file mode 100644 index 9e8def483d8..00000000000 --- a/salt/ext/tornado/speedups.pyi +++ /dev/null @@ -1 +0,0 @@ -def websocket_mask(mask: bytes, data: bytes) -> bytes: ... diff --git a/salt/ext/tornado/stack_context.py b/salt/ext/tornado/stack_context.py deleted file mode 100644 index 44da43a969a..00000000000 --- a/salt/ext/tornado/stack_context.py +++ /dev/null @@ -1,391 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2010 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""`StackContext` allows applications to maintain threadlocal-like state -that follows execution as it moves to other execution contexts. - -The motivating examples are to eliminate the need for explicit -``async_callback`` wrappers (as in `tornado.web.RequestHandler`), and to -allow some additional context to be kept for logging. - -This is slightly magic, but it's an extension of the idea that an -exception handler is a kind of stack-local state and when that stack -is suspended and resumed in a new context that state needs to be -preserved. `StackContext` shifts the burden of restoring that state -from each call site (e.g. wrapping each `.AsyncHTTPClient` callback -in ``async_callback``) to the mechanisms that transfer control from -one context to another (e.g. `.AsyncHTTPClient` itself, `.IOLoop`, -thread pools, etc). - -Example usage:: - - @contextlib.contextmanager - def die_on_error(): - try: - yield - except Exception: - logging.error("exception in asynchronous operation",exc_info=True) - sys.exit(1) - - with StackContext(die_on_error): - # Any exception thrown here *or in callback and its descendants* - # will cause the process to exit instead of spinning endlessly - # in the ioloop. - http_client.fetch(url, callback) - ioloop.start() - -Most applications shouldn't have to work with `StackContext` directly. -Here are a few rules of thumb for when it's necessary: - -* If you're writing an asynchronous library that doesn't rely on a - stack_context-aware library like `tornado.ioloop` or `tornado.iostream` - (for example, if you're writing a thread pool), use - `.stack_context.wrap()` before any asynchronous operations to capture the - stack context from where the operation was started. - -* If you're writing an asynchronous library that has some shared - resources (such as a connection pool), create those shared resources - within a ``with stack_context.NullContext():`` block. This will prevent - ``StackContexts`` from leaking from one request to another. - -* If you want to write something like an exception handler that will - persist across asynchronous calls, create a new `StackContext` (or - `ExceptionStackContext`), and make your asynchronous calls in a ``with`` - block that references your `StackContext`. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import sys -import threading - -from salt.ext.tornado.util import raise_exc_info - - -class StackContextInconsistentError(Exception): - pass - - -class _State(threading.local): - def __init__(self): - self.contexts = (tuple(), None) - - -_state = _State() - - -class StackContext(object): - """Establishes the given context as a StackContext that will be transferred. - - Note that the parameter is a callable that returns a context - manager, not the context itself. That is, where for a - non-transferable context manager you would say:: - - with my_context(): - - StackContext takes the function itself rather than its result:: - - with StackContext(my_context): - - The result of ``with StackContext() as cb:`` is a deactivation - callback. Run this callback when the StackContext is no longer - needed to ensure that it is not propagated any further (note that - deactivating a context does not affect any instances of that - context that are currently pending). This is an advanced feature - and not necessary in most applications. - """ - def __init__(self, context_factory): - self.context_factory = context_factory - self.contexts = [] - self.active = True - - def _deactivate(self): - self.active = False - - # StackContext protocol - def enter(self): - context = self.context_factory() - self.contexts.append(context) - context.__enter__() - - def exit(self, type, value, traceback): - context = self.contexts.pop() - context.__exit__(type, value, traceback) - - # Note that some of this code is duplicated in ExceptionStackContext - # below. ExceptionStackContext is more common and doesn't need - # the full generality of this class. - def __enter__(self): - self.old_contexts = _state.contexts - self.new_contexts = (self.old_contexts[0] + (self,), self) - _state.contexts = self.new_contexts - - try: - self.enter() - except: - _state.contexts = self.old_contexts - raise - - return self._deactivate - - def __exit__(self, type, value, traceback): - try: - self.exit(type, value, traceback) - finally: - final_contexts = _state.contexts - _state.contexts = self.old_contexts - - # Generator coroutines and with-statements with non-local - # effects interact badly. Check here for signs of - # the stack getting out of sync. - # Note that this check comes after restoring _state.context - # so that if it fails things are left in a (relatively) - # consistent state. - if final_contexts is not self.new_contexts: - raise StackContextInconsistentError( - 'stack_context inconsistency (may be caused by yield ' - 'within a "with StackContext" block)') - - # Break up a reference to itself to allow for faster GC on CPython. - self.new_contexts = None - - -class ExceptionStackContext(object): - """Specialization of StackContext for exception handling. - - The supplied ``exception_handler`` function will be called in the - event of an uncaught exception in this context. The semantics are - similar to a try/finally clause, and intended use cases are to log - an error, close a socket, or similar cleanup actions. The - ``exc_info`` triple ``(type, value, traceback)`` will be passed to the - exception_handler function. - - If the exception handler returns true, the exception will be - consumed and will not be propagated to other exception handlers. - """ - def __init__(self, exception_handler): - self.exception_handler = exception_handler - self.active = True - - def _deactivate(self): - self.active = False - - def exit(self, type, value, traceback): - if type is not None: - return self.exception_handler(type, value, traceback) - - def __enter__(self): - self.old_contexts = _state.contexts - self.new_contexts = (self.old_contexts[0], self) - _state.contexts = self.new_contexts - - return self._deactivate - - def __exit__(self, type, value, traceback): - try: - if type is not None: - return self.exception_handler(type, value, traceback) - finally: - final_contexts = _state.contexts - _state.contexts = self.old_contexts - - if final_contexts is not self.new_contexts: - raise StackContextInconsistentError( - 'stack_context inconsistency (may be caused by yield ' - 'within a "with StackContext" block)') - - # Break up a reference to itself to allow for faster GC on CPython. - self.new_contexts = None - - -class NullContext(object): - """Resets the `StackContext`. - - Useful when creating a shared resource on demand (e.g. an - `.AsyncHTTPClient`) where the stack that caused the creating is - not relevant to future operations. - """ - def __enter__(self): - self.old_contexts = _state.contexts - _state.contexts = (tuple(), None) - - def __exit__(self, type, value, traceback): - _state.contexts = self.old_contexts - - -def _remove_deactivated(contexts): - """Remove deactivated handlers from the chain""" - # Clean ctx handlers - stack_contexts = tuple([h for h in contexts[0] if h.active]) - - # Find new head - head = contexts[1] - while head is not None and not head.active: - head = head.old_contexts[1] - - # Process chain - ctx = head - while ctx is not None: - parent = ctx.old_contexts[1] - - while parent is not None: - if parent.active: - break - ctx.old_contexts = parent.old_contexts - parent = parent.old_contexts[1] - - ctx = parent - - return (stack_contexts, head) - - -def wrap(fn): - """Returns a callable object that will restore the current `StackContext` - when executed. - - Use this whenever saving a callback to be executed later in a - different execution context (either in a different thread or - asynchronously in the same thread). - """ - # Check if function is already wrapped - if fn is None or hasattr(fn, '_wrapped'): - return fn - - # Capture current stack head - # TODO: Any other better way to store contexts and update them in wrapped function? - cap_contexts = [_state.contexts] - - if not cap_contexts[0][0] and not cap_contexts[0][1]: - # Fast path when there are no active contexts. - def null_wrapper(*args, **kwargs): - try: - current_state = _state.contexts - _state.contexts = cap_contexts[0] - return fn(*args, **kwargs) - finally: - _state.contexts = current_state - null_wrapper._wrapped = True - return null_wrapper - - def wrapped(*args, **kwargs): - ret = None - try: - # Capture old state - current_state = _state.contexts - - # Remove deactivated items - cap_contexts[0] = contexts = _remove_deactivated(cap_contexts[0]) - - # Force new state - _state.contexts = contexts - - # Current exception - exc = (None, None, None) - top = None - - # Apply stack contexts - last_ctx = 0 - stack = contexts[0] - - # Apply state - for n in stack: - try: - n.enter() - last_ctx += 1 - except: - # Exception happened. Record exception info and store top-most handler - exc = sys.exc_info() - top = n.old_contexts[1] - - # Execute callback if no exception happened while restoring state - if top is None: - try: - ret = fn(*args, **kwargs) - except: - exc = sys.exc_info() - top = contexts[1] - - # If there was exception, try to handle it by going through the exception chain - if top is not None: - exc = _handle_exception(top, exc) - else: - # Otherwise take shorter path and run stack contexts in reverse order - while last_ctx > 0: - last_ctx -= 1 - c = stack[last_ctx] - - try: - c.exit(*exc) - except: - exc = sys.exc_info() - top = c.old_contexts[1] - break - else: - top = None - - # If if exception happened while unrolling, take longer exception handler path - if top is not None: - exc = _handle_exception(top, exc) - - # If exception was not handled, raise it - if exc != (None, None, None): - raise_exc_info(exc) - finally: - _state.contexts = current_state - return ret - - wrapped._wrapped = True - return wrapped - - -def _handle_exception(tail, exc): - while tail is not None: - try: - if tail.exit(*exc): - exc = (None, None, None) - except: - exc = sys.exc_info() - - tail = tail.old_contexts[1] - - return exc - - -def run_with_stack_context(context, func): - """Run a coroutine ``func`` in the given `StackContext`. - - It is not safe to have a ``yield`` statement within a ``with StackContext`` - block, so it is difficult to use stack context with `.gen.coroutine`. - This helper function runs the function in the correct context while - keeping the ``yield`` and ``with`` statements syntactically separate. - - Example:: - - @gen.coroutine - def incorrect(): - with StackContext(ctx): - # ERROR: this will raise StackContextInconsistentError - yield other_coroutine() - - @gen.coroutine - def correct(): - yield run_with_stack_context(StackContext(ctx), other_coroutine) - - .. versionadded:: 3.1 - """ - with context: - return func() diff --git a/salt/ext/tornado/tcpclient.py b/salt/ext/tornado/tcpclient.py deleted file mode 100644 index abd41a95313..00000000000 --- a/salt/ext/tornado/tcpclient.py +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2014 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A non-blocking TCP connection factory. -""" -# pylint: skip-file -from __future__ import absolute_import, division, print_function - -import functools -import socket - -from salt.ext.tornado.concurrent import Future -from salt.ext.tornado.ioloop import IOLoop -from salt.ext.tornado.iostream import IOStream -from salt.ext.tornado import gen -from salt.ext.tornado.netutil import Resolver -from salt.ext.tornado.platform.auto import set_close_exec - -_INITIAL_CONNECT_TIMEOUT = 0.3 - - -class _Connector(object): - """A stateless implementation of the "Happy Eyeballs" algorithm. - - "Happy Eyeballs" is documented in RFC6555 as the recommended practice - for when both IPv4 and IPv6 addresses are available. - - In this implementation, we partition the addresses by family, and - make the first connection attempt to whichever address was - returned first by ``getaddrinfo``. If that connection fails or - times out, we begin a connection in parallel to the first address - of the other family. If there are additional failures we retry - with other addresses, keeping one connection attempt per family - in flight at a time. - - http://tools.ietf.org/html/rfc6555 - - """ - def __init__(self, addrinfo, io_loop, connect): - self.io_loop = io_loop - self.connect = connect - - self.future = Future() - self.timeout = None - self.last_error = None - self.remaining = len(addrinfo) - self.primary_addrs, self.secondary_addrs = self.split(addrinfo) - - @staticmethod - def split(addrinfo): - """Partition the ``addrinfo`` list by address family. - - Returns two lists. The first list contains the first entry from - ``addrinfo`` and all others with the same family, and the - second list contains all other addresses (normally one list will - be AF_INET and the other AF_INET6, although non-standard resolvers - may return additional families). - """ - primary = [] - secondary = [] - primary_af = addrinfo[0][0] - for af, addr in addrinfo: - if af == primary_af: - primary.append((af, addr)) - else: - secondary.append((af, addr)) - return primary, secondary - - def start(self, timeout=_INITIAL_CONNECT_TIMEOUT): - self.try_connect(iter(self.primary_addrs)) - self.set_timout(timeout) - return self.future - - def try_connect(self, addrs): - try: - af, addr = next(addrs) - except StopIteration: - # We've reached the end of our queue, but the other queue - # might still be working. Send a final error on the future - # only when both queues are finished. - if self.remaining == 0 and not self.future.done(): - self.future.set_exception(self.last_error or - IOError("connection failed")) - return - future = self.connect(af, addr) - future.add_done_callback(functools.partial(self.on_connect_done, - addrs, af, addr)) - - def on_connect_done(self, addrs, af, addr, future): - self.remaining -= 1 - try: - stream = future.result() - except Exception as e: - if self.future.done(): - return - # Error: try again (but remember what happened so we have an - # error to raise in the end) - self.last_error = e - self.try_connect(addrs) - if self.timeout is not None: - # If the first attempt failed, don't wait for the - # timeout to try an address from the secondary queue. - self.io_loop.remove_timeout(self.timeout) - self.on_timeout() - return - self.clear_timeout() - if self.future.done(): - # This is a late arrival; just drop it. - stream.close() - else: - self.future.set_result((af, addr, stream)) - - def set_timout(self, timeout): - self.timeout = self.io_loop.add_timeout(self.io_loop.time() + timeout, - self.on_timeout) - - def on_timeout(self): - self.timeout = None - self.try_connect(iter(self.secondary_addrs)) - - def clear_timeout(self): - if self.timeout is not None: - self.io_loop.remove_timeout(self.timeout) - - -class TCPClient(object): - """A non-blocking TCP connection factory. - - .. versionchanged:: 4.1 - The ``io_loop`` argument is deprecated. - """ - def __init__(self, resolver=None, io_loop=None): - self.io_loop = io_loop or IOLoop.current() - if resolver is not None: - self.resolver = resolver - self._own_resolver = False - else: - self.resolver = Resolver(io_loop=io_loop) - self._own_resolver = True - - def close(self): - if self._own_resolver: - self.resolver.close() - - @gen.coroutine - def connect(self, host, port, af=socket.AF_UNSPEC, ssl_options=None, - max_buffer_size=None, source_ip=None, source_port=None): - """Connect to the given host and port. - - Asynchronously returns an `.IOStream` (or `.SSLIOStream` if - ``ssl_options`` is not None). - - Using the ``source_ip`` kwarg, one can specify the source - IP address to use when establishing the connection. - In case the user needs to resolve and - use a specific interface, it has to be handled outside - of Tornado as this depends very much on the platform. - - Similarly, when the user requires a certain source port, it can - be specified using the ``source_port`` arg. - - .. versionchanged:: 4.5 - Added the ``source_ip`` and ``source_port`` arguments. - """ - addrinfo = yield self.resolver.resolve(host, port, af) - connector = _Connector( - addrinfo, self.io_loop, - functools.partial(self._create_stream, max_buffer_size, - source_ip=source_ip, source_port=source_port) - ) - af, addr, stream = yield connector.start() - # TODO: For better performance we could cache the (af, addr) - # information here and re-use it on subsequent connections to - # the same host. (http://tools.ietf.org/html/rfc6555#section-4.2) - if ssl_options is not None: - stream = yield stream.start_tls(False, ssl_options=ssl_options, - server_hostname=host) - raise gen.Return(stream) - - def _create_stream(self, max_buffer_size, af, addr, source_ip=None, - source_port=None): - # Always connect in plaintext; we'll convert to ssl if necessary - # after one connection has completed. - source_port_bind = source_port if isinstance(source_port, int) else 0 - source_ip_bind = source_ip - if source_port_bind and not source_ip: - # User required a specific port, but did not specify - # a certain source IP, will bind to the default loopback. - source_ip_bind = '::1' if af == socket.AF_INET6 else '127.0.0.1' - # Trying to use the same address family as the requested af socket: - # - 127.0.0.1 for IPv4 - # - ::1 for IPv6 - socket_obj = socket.socket(af) - set_close_exec(socket_obj.fileno()) - if source_port_bind or source_ip_bind: - # If the user requires binding also to a specific IP/port. - try: - socket_obj.bind((source_ip_bind, source_port_bind)) - except socket.error: - socket_obj.close() - # Fail loudly if unable to use the IP/port. - raise - try: - stream = IOStream(socket_obj, - io_loop=self.io_loop, - max_buffer_size=max_buffer_size) - except socket.error as e: - fu = Future() - fu.set_exception(e) - return fu - else: - return stream.connect(addr) diff --git a/salt/ext/tornado/tcpserver.py b/salt/ext/tornado/tcpserver.py deleted file mode 100644 index 3b40d08c13b..00000000000 --- a/salt/ext/tornado/tcpserver.py +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2011 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A non-blocking, single-threaded TCP server.""" -# pylint: skip-file -from __future__ import absolute_import, division, print_function - -import errno -import os -import socket - -from salt.ext.tornado import gen -from salt.ext.tornado.log import app_log -from salt.ext.tornado.ioloop import IOLoop -from salt.ext.tornado.iostream import IOStream, SSLIOStream -from salt.ext.tornado.netutil import bind_sockets, add_accept_handler, ssl_wrap_socket -from salt.ext.tornado import process -from salt.ext.tornado.util import errno_from_exception - -try: - import ssl -except ImportError: - # ssl is not available on Google App Engine. - ssl = None - - -class TCPServer(object): - r"""A non-blocking, single-threaded TCP server. - - To use `TCPServer`, define a subclass which overrides the `handle_stream` - method. For example, a simple echo server could be defined like this:: - - from salt.ext.tornado.tcpserver import TCPServer - from salt.ext.tornado.iostream import StreamClosedError - from salt.ext.tornado import gen - - class EchoServer(TCPServer): - @gen.coroutine - def handle_stream(self, stream, address): - while True: - try: - data = yield stream.read_until(b"\n") - yield stream.write(data) - except StreamClosedError: - break - - To make this server serve SSL traffic, send the ``ssl_options`` keyword - argument with an `ssl.SSLContext` object. For compatibility with older - versions of Python ``ssl_options`` may also be a dictionary of keyword - arguments for the `ssl.wrap_socket` method.:: - - ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ssl_ctx.load_cert_chain(os.path.join(data_dir, "mydomain.crt"), - os.path.join(data_dir, "mydomain.key")) - TCPServer(ssl_options=ssl_ctx) - - `TCPServer` initialization follows one of three patterns: - - 1. `listen`: simple single-process:: - - server = TCPServer() - server.listen(8888) - IOLoop.current().start() - - 2. `bind`/`start`: simple multi-process:: - - server = TCPServer() - server.bind(8888) - server.start(0) # Forks multiple sub-processes - IOLoop.current().start() - - When using this interface, an `.IOLoop` must *not* be passed - to the `TCPServer` constructor. `start` will always start - the server on the default singleton `.IOLoop`. - - 3. `add_sockets`: advanced multi-process:: - - sockets = bind_sockets(8888) - tornado.process.fork_processes(0) - server = TCPServer() - server.add_sockets(sockets) - IOLoop.current().start() - - The `add_sockets` interface is more complicated, but it can be - used with `tornado.process.fork_processes` to give you more - flexibility in when the fork happens. `add_sockets` can - also be used in single-process servers if you want to create - your listening sockets in some way other than - `~tornado.netutil.bind_sockets`. - - .. versionadded:: 3.1 - The ``max_buffer_size`` argument. - """ - def __init__(self, io_loop=None, ssl_options=None, max_buffer_size=None, - read_chunk_size=None): - self.io_loop = io_loop - self.ssl_options = ssl_options - self._sockets = {} # fd -> socket object - self._pending_sockets = [] - self._started = False - self._stopped = False - self.max_buffer_size = max_buffer_size - self.read_chunk_size = read_chunk_size - - # Verify the SSL options. Otherwise we don't get errors until clients - # connect. This doesn't verify that the keys are legitimate, but - # the SSL module doesn't do that until there is a connected socket - # which seems like too much work - if self.ssl_options is not None and isinstance(self.ssl_options, dict): - # Only certfile is required: it can contain both keys - if 'certfile' not in self.ssl_options: - raise KeyError('missing key "certfile" in ssl_options') - - if not os.path.exists(self.ssl_options['certfile']): - raise ValueError('certfile "%s" does not exist' % - self.ssl_options['certfile']) - if ('keyfile' in self.ssl_options and - not os.path.exists(self.ssl_options['keyfile'])): - raise ValueError('keyfile "%s" does not exist' % - self.ssl_options['keyfile']) - - def listen(self, port, address=""): - """Starts accepting connections on the given port. - - This method may be called more than once to listen on multiple ports. - `listen` takes effect immediately; it is not necessary to call - `TCPServer.start` afterwards. It is, however, necessary to start - the `.IOLoop`. - """ - sockets = bind_sockets(port, address=address) - self.add_sockets(sockets) - - def add_sockets(self, sockets): - """Makes this server start accepting connections on the given sockets. - - The ``sockets`` parameter is a list of socket objects such as - those returned by `~tornado.netutil.bind_sockets`. - `add_sockets` is typically used in combination with that - method and `tornado.process.fork_processes` to provide greater - control over the initialization of a multi-process server. - """ - if self.io_loop is None: - self.io_loop = IOLoop.current() - - for sock in sockets: - self._sockets[sock.fileno()] = sock - add_accept_handler(sock, self._handle_connection, - io_loop=self.io_loop) - - def add_socket(self, socket): - """Singular version of `add_sockets`. Takes a single socket object.""" - self.add_sockets([socket]) - - def bind(self, port, address=None, family=socket.AF_UNSPEC, backlog=128, - reuse_port=False): - """Binds this server to the given port on the given address. - - To start the server, call `start`. If you want to run this server - in a single process, you can call `listen` as a shortcut to the - sequence of `bind` and `start` calls. - - Address may be either an IP address or hostname. If it's a hostname, - the server will listen on all IP addresses associated with the - name. Address may be an empty string or None to listen on all - available interfaces. Family may be set to either `socket.AF_INET` - or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise - both will be used if available. - - The ``backlog`` argument has the same meaning as for - `socket.listen `. The ``reuse_port`` argument - has the same meaning as for `.bind_sockets`. - - This method may be called multiple times prior to `start` to listen - on multiple ports or interfaces. - - .. versionchanged:: 4.4 - Added the ``reuse_port`` argument. - """ - sockets = bind_sockets(port, address=address, family=family, - backlog=backlog, reuse_port=reuse_port) - if self._started: - self.add_sockets(sockets) - else: - self._pending_sockets.extend(sockets) - - def start(self, num_processes=1): - """Starts this server in the `.IOLoop`. - - By default, we run the server in this process and do not fork any - additional child process. - - If num_processes is ``None`` or <= 0, we detect the number of cores - available on this machine and fork that number of child - processes. If num_processes is given and > 1, we fork that - specific number of sub-processes. - - Since we use processes and not threads, there is no shared memory - between any server code. - - Note that multiple processes are not compatible with the autoreload - module (or the ``autoreload=True`` option to `tornado.web.Application` - which defaults to True when ``debug=True``). - When using multiple processes, no IOLoops can be created or - referenced until after the call to ``TCPServer.start(n)``. - """ - assert not self._started - self._started = True - if num_processes != 1: - process.fork_processes(num_processes) - sockets = self._pending_sockets - self._pending_sockets = [] - self.add_sockets(sockets) - - def stop(self): - """Stops listening for new connections. - - Requests currently in progress may still continue after the - server is stopped. - """ - if self._stopped: - return - self._stopped = True - for fd, sock in self._sockets.items(): - assert sock.fileno() == fd - self.io_loop.remove_handler(fd) - sock.close() - - def handle_stream(self, stream, address): - """Override to handle a new `.IOStream` from an incoming connection. - - This method may be a coroutine; if so any exceptions it raises - asynchronously will be logged. Accepting of incoming connections - will not be blocked by this coroutine. - - If this `TCPServer` is configured for SSL, ``handle_stream`` - may be called before the SSL handshake has completed. Use - `.SSLIOStream.wait_for_handshake` if you need to verify the client's - certificate or use NPN/ALPN. - - .. versionchanged:: 4.2 - Added the option for this method to be a coroutine. - """ - raise NotImplementedError() - - def _handle_connection(self, connection, address): - if self.ssl_options is not None: - assert ssl, "Python 2.6+ and OpenSSL required for SSL" - try: - connection = ssl_wrap_socket(connection, - self.ssl_options, - server_side=True, - do_handshake_on_connect=False) - except ssl.SSLError as err: - if err.args[0] == ssl.SSL_ERROR_EOF: - return connection.close() - else: - raise - except socket.error as err: - # If the connection is closed immediately after it is created - # (as in a port scan), we can get one of several errors. - # wrap_socket makes an internal call to getpeername, - # which may return either EINVAL (Mac OS X) or ENOTCONN - # (Linux). If it returns ENOTCONN, this error is - # silently swallowed by the ssl module, so we need to - # catch another error later on (AttributeError in - # SSLIOStream._do_ssl_handshake). - # To test this behavior, try nmap with the -sT flag. - # https://github.com/tornadoweb/tornado/pull/750 - if errno_from_exception(err) in (errno.ECONNABORTED, errno.EINVAL): - return connection.close() - else: - raise - try: - if self.ssl_options is not None: - stream = SSLIOStream(connection, io_loop=self.io_loop, - max_buffer_size=self.max_buffer_size, - read_chunk_size=self.read_chunk_size) - else: - stream = IOStream(connection, io_loop=self.io_loop, - max_buffer_size=self.max_buffer_size, - read_chunk_size=self.read_chunk_size) - - future = self.handle_stream(stream, address) - if future is not None: - self.io_loop.add_future(gen.convert_yielded(future), - lambda f: f.result()) - except Exception: - app_log.error("Error in connection callback", exc_info=True) diff --git a/salt/ext/tornado/template.py b/salt/ext/tornado/template.py deleted file mode 100644 index 8d3818b1c28..00000000000 --- a/salt/ext/tornado/template.py +++ /dev/null @@ -1,979 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A simple template system that compiles templates to Python code. - -Basic usage looks like:: - - t = template.Template("{{ myvalue }}") - print(t.generate(myvalue="XXX")) - -`Loader` is a class that loads templates from a root directory and caches -the compiled templates:: - - loader = template.Loader("/home/btaylor") - print(loader.load("test.html").generate(myvalue="XXX")) - -We compile all templates to raw Python. Error-reporting is currently... uh, -interesting. Syntax for the templates:: - - ### base.html - - - {% block title %}Default title{% end %} - - -
    - {% for student in students %} - {% block student %} -
  • {{ escape(student.name) }}
    • - {% end %} - {% end %} -
- - - - ### bold.html - {% extends "base.html" %} - - {% block title %}A bolder title{% end %} - - {% block student %} -
  • {{ escape(student.name) }}
    • - {% end %} - -Unlike most other template systems, we do not put any restrictions on the -expressions you can include in your statements. ``if`` and ``for`` blocks get -translated exactly into Python, so you can do complex expressions like:: - - {% for student in [p for p in people if p.student and p.age > 23] %} -
  • {{ escape(student.name) }}
    • - {% end %} - -Translating directly to Python means you can apply functions to expressions -easily, like the ``escape()`` function in the examples above. You can pass -functions in to your template just like any other variable -(In a `.RequestHandler`, override `.RequestHandler.get_template_namespace`):: - - ### Python code - def add(x, y): - return x + y - template.execute(add=add) - - ### The template - {{ add(1, 2) }} - -We provide the functions `escape() <.xhtml_escape>`, `.url_escape()`, -`.json_encode()`, and `.squeeze()` to all templates by default. - -Typical applications do not create `Template` or `Loader` instances by -hand, but instead use the `~.RequestHandler.render` and -`~.RequestHandler.render_string` methods of -`tornado.web.RequestHandler`, which load templates automatically based -on the ``template_path`` `.Application` setting. - -Variable names beginning with ``_tt_`` are reserved by the template -system and should not be used by application code. - -Syntax Reference ----------------- - -Template expressions are surrounded by double curly braces: ``{{ ... }}``. -The contents may be any python expression, which will be escaped according -to the current autoescape setting and inserted into the output. Other -template directives use ``{% %}``. - -To comment out a section so that it is omitted from the output, surround it -with ``{# ... #}``. - -These tags may be escaped as ``{{!``, ``{%!``, and ``{#!`` -if you need to include a literal ``{{``, ``{%``, or ``{#`` in the output. - - -``{% apply *function* %}...{% end %}`` - Applies a function to the output of all template code between ``apply`` - and ``end``:: - - {% apply linkify %}{{name}} said: {{message}}{% end %} - - Note that as an implementation detail apply blocks are implemented - as nested functions and thus may interact strangely with variables - set via ``{% set %}``, or the use of ``{% break %}`` or ``{% continue %}`` - within loops. - -``{% autoescape *function* %}`` - Sets the autoescape mode for the current file. This does not affect - other files, even those referenced by ``{% include %}``. Note that - autoescaping can also be configured globally, at the `.Application` - or `Loader`.:: - - {% autoescape xhtml_escape %} - {% autoescape None %} - -``{% block *name* %}...{% end %}`` - Indicates a named, replaceable block for use with ``{% extends %}``. - Blocks in the parent template will be replaced with the contents of - the same-named block in a child template.:: - - - {% block title %}Default title{% end %} - - - {% extends "base.html" %} - {% block title %}My page title{% end %} - -``{% comment ... %}`` - A comment which will be removed from the template output. Note that - there is no ``{% end %}`` tag; the comment goes from the word ``comment`` - to the closing ``%}`` tag. - -``{% extends *filename* %}`` - Inherit from another template. Templates that use ``extends`` should - contain one or more ``block`` tags to replace content from the parent - template. Anything in the child template not contained in a ``block`` - tag will be ignored. For an example, see the ``{% block %}`` tag. - -``{% for *var* in *expr* %}...{% end %}`` - Same as the python ``for`` statement. ``{% break %}`` and - ``{% continue %}`` may be used inside the loop. - -``{% from *x* import *y* %}`` - Same as the python ``import`` statement. - -``{% if *condition* %}...{% elif *condition* %}...{% else %}...{% end %}`` - Conditional statement - outputs the first section whose condition is - true. (The ``elif`` and ``else`` sections are optional) - -``{% import *module* %}`` - Same as the python ``import`` statement. - -``{% include *filename* %}`` - Includes another template file. The included file can see all the local - variables as if it were copied directly to the point of the ``include`` - directive (the ``{% autoescape %}`` directive is an exception). - Alternately, ``{% module Template(filename, **kwargs) %}`` may be used - to include another template with an isolated namespace. - -``{% module *expr* %}`` - Renders a `~tornado.web.UIModule`. The output of the ``UIModule`` is - not escaped:: - - {% module Template("foo.html", arg=42) %} - - ``UIModules`` are a feature of the `tornado.web.RequestHandler` - class (and specifically its ``render`` method) and will not work - when the template system is used on its own in other contexts. - -``{% raw *expr* %}`` - Outputs the result of the given expression without autoescaping. - -``{% set *x* = *y* %}`` - Sets a local variable. - -``{% try %}...{% except %}...{% else %}...{% finally %}...{% end %}`` - Same as the python ``try`` statement. - -``{% while *condition* %}... {% end %}`` - Same as the python ``while`` statement. ``{% break %}`` and - ``{% continue %}`` may be used inside the loop. - -``{% whitespace *mode* %}`` - Sets the whitespace mode for the remainder of the current file - (or until the next ``{% whitespace %}`` directive). See - `filter_whitespace` for available options. New in Tornado 4.3. -""" -# pylint: skip-file - -from __future__ import absolute_import, division, print_function - -import datetime -import linecache -import os.path -import posixpath -import re -import threading - -from salt.ext.tornado import escape -from salt.ext.tornado.log import app_log -from salt.ext.tornado.util import ObjectDict, exec_in, unicode_type, PY3 - -if PY3: - from io import StringIO -else: - from cStringIO import StringIO - -_DEFAULT_AUTOESCAPE = "xhtml_escape" -_UNSET = object() - - -def filter_whitespace(mode, text): - """Transform whitespace in ``text`` according to ``mode``. - - Available modes are: - - * ``all``: Return all whitespace unmodified. - * ``single``: Collapse consecutive whitespace with a single whitespace - character, preserving newlines. - * ``oneline``: Collapse all runs of whitespace into a single space - character, removing all newlines in the process. - - .. versionadded:: 4.3 - """ - if mode == 'all': - return text - elif mode == 'single': - text = re.sub(r"([\t ]+)", " ", text) - text = re.sub(r"(\s*\n\s*)", "\n", text) - return text - elif mode == 'oneline': - return re.sub(r"(\s+)", " ", text) - else: - raise Exception("invalid whitespace mode %s" % mode) - - -class Template(object): - """A compiled template. - - We compile into Python from the given template_string. You can generate - the template from variables with generate(). - """ - # note that the constructor's signature is not extracted with - # autodoc because _UNSET looks like garbage. When changing - # this signature update website/sphinx/template.rst too. - def __init__(self, template_string, name="", loader=None, - compress_whitespace=_UNSET, autoescape=_UNSET, - whitespace=None): - """Construct a Template. - - :arg str template_string: the contents of the template file. - :arg str name: the filename from which the template was loaded - (used for error message). - :arg tornado.template.BaseLoader loader: the `~tornado.template.BaseLoader` responsible for this template, - used to resolve ``{% include %}`` and ``{% extend %}`` - directives. - :arg bool compress_whitespace: Deprecated since Tornado 4.3. - Equivalent to ``whitespace="single"`` if true and - ``whitespace="all"`` if false. - :arg str autoescape: The name of a function in the template - namespace, or ``None`` to disable escaping by default. - :arg str whitespace: A string specifying treatment of whitespace; - see `filter_whitespace` for options. - - .. versionchanged:: 4.3 - Added ``whitespace`` parameter; deprecated ``compress_whitespace``. - """ - self.name = escape.native_str(name) - - if compress_whitespace is not _UNSET: - # Convert deprecated compress_whitespace (bool) to whitespace (str). - if whitespace is not None: - raise Exception("cannot set both whitespace and compress_whitespace") - whitespace = "single" if compress_whitespace else "all" - if whitespace is None: - if loader and loader.whitespace: - whitespace = loader.whitespace - else: - # Whitespace defaults by filename. - if name.endswith(".html") or name.endswith(".js"): - whitespace = "single" - else: - whitespace = "all" - # Validate the whitespace setting. - filter_whitespace(whitespace, '') - - if autoescape is not _UNSET: - self.autoescape = autoescape - elif loader: - self.autoescape = loader.autoescape - else: - self.autoescape = _DEFAULT_AUTOESCAPE - - self.namespace = loader.namespace if loader else {} - reader = _TemplateReader(name, escape.native_str(template_string), - whitespace) - self.file = _File(self, _parse(reader, self)) - self.code = self._generate_python(loader) - self.loader = loader - try: - # Under python2.5, the fake filename used here must match - # the module name used in __name__ below. - # The dont_inherit flag prevents template.py's future imports - # from being applied to the generated code. - self.compiled = compile( - escape.to_unicode(self.code), - "%s.generated.py" % self.name.replace('.', '_'), - "exec", dont_inherit=True) - except Exception: - formatted_code = _format_code(self.code).rstrip() - app_log.error("%s code:\n%s", self.name, formatted_code) - raise - - def generate(self, **kwargs): - """Generate this template with the given arguments.""" - namespace = { - "escape": escape.xhtml_escape, - "xhtml_escape": escape.xhtml_escape, - "url_escape": escape.url_escape, - "json_encode": escape.json_encode, - "squeeze": escape.squeeze, - "linkify": escape.linkify, - "datetime": datetime, - "_tt_utf8": escape.utf8, # for internal use - "_tt_string_types": (unicode_type, bytes), - # __name__ and __loader__ allow the traceback mechanism to find - # the generated source code. - "__name__": self.name.replace('.', '_'), - "__loader__": ObjectDict(get_source=lambda name: self.code), - } - namespace.update(self.namespace) - namespace.update(kwargs) - exec_in(self.compiled, namespace) - execute = namespace["_tt_execute"] - # Clear the traceback module's cache of source data now that - # we've generated a new template (mainly for this module's - # unittests, where different tests reuse the same name). - linecache.clearcache() - return execute() - - def _generate_python(self, loader): - buffer = StringIO() - try: - # named_blocks maps from names to _NamedBlock objects - named_blocks = {} - ancestors = self._get_ancestors(loader) - ancestors.reverse() - for ancestor in ancestors: - ancestor.find_named_blocks(loader, named_blocks) - writer = _CodeWriter(buffer, named_blocks, loader, - ancestors[0].template) - ancestors[0].generate(writer) - return buffer.getvalue() - finally: - buffer.close() - - def _get_ancestors(self, loader): - ancestors = [self.file] - for chunk in self.file.body.chunks: - if isinstance(chunk, _ExtendsBlock): - if not loader: - raise ParseError("{% extends %} block found, but no " - "template loader") - template = loader.load(chunk.name, self.name) - ancestors.extend(template._get_ancestors(loader)) - return ancestors - - -class BaseLoader(object): - """Base class for template loaders. - - You must use a template loader to use template constructs like - ``{% extends %}`` and ``{% include %}``. The loader caches all - templates after they are loaded the first time. - """ - def __init__(self, autoescape=_DEFAULT_AUTOESCAPE, namespace=None, - whitespace=None): - """Construct a template loader. - - :arg str autoescape: The name of a function in the template - namespace, such as "xhtml_escape", or ``None`` to disable - autoescaping by default. - :arg dict namespace: A dictionary to be added to the default template - namespace, or ``None``. - :arg str whitespace: A string specifying default behavior for - whitespace in templates; see `filter_whitespace` for options. - Default is "single" for files ending in ".html" and ".js" and - "all" for other files. - - .. versionchanged:: 4.3 - Added ``whitespace`` parameter. - """ - self.autoescape = autoescape - self.namespace = namespace or {} - self.whitespace = whitespace - self.templates = {} - # self.lock protects self.templates. It's a reentrant lock - # because templates may load other templates via `include` or - # `extends`. Note that thanks to the GIL this code would be safe - # even without the lock, but could lead to wasted work as multiple - # threads tried to compile the same template simultaneously. - self.lock = threading.RLock() - - def reset(self): - """Resets the cache of compiled templates.""" - with self.lock: - self.templates = {} - - def resolve_path(self, name, parent_path=None): - """Converts a possibly-relative path to absolute (used internally).""" - raise NotImplementedError() - - def load(self, name, parent_path=None): - """Loads a template.""" - name = self.resolve_path(name, parent_path=parent_path) - with self.lock: - if name not in self.templates: - self.templates[name] = self._create_template(name) - return self.templates[name] - - def _create_template(self, name): - raise NotImplementedError() - - -class Loader(BaseLoader): - """A template loader that loads from a single root directory. - """ - def __init__(self, root_directory, **kwargs): - super(Loader, self).__init__(**kwargs) - self.root = os.path.abspath(root_directory) - - def resolve_path(self, name, parent_path=None): - if parent_path and not parent_path.startswith("<") and \ - not parent_path.startswith("/") and \ - not name.startswith("/"): - current_path = os.path.join(self.root, parent_path) - file_dir = os.path.dirname(os.path.abspath(current_path)) - relative_path = os.path.abspath(os.path.join(file_dir, name)) - if relative_path.startswith(self.root): - name = relative_path[len(self.root) + 1:] - return name - - def _create_template(self, name): - path = os.path.join(self.root, name) - with open(path, "rb") as f: - template = Template(f.read(), name=name, loader=self) - return template - - -class DictLoader(BaseLoader): - """A template loader that loads from a dictionary.""" - def __init__(self, dict, **kwargs): - super(DictLoader, self).__init__(**kwargs) - self.dict = dict - - def resolve_path(self, name, parent_path=None): - if parent_path and not parent_path.startswith("<") and \ - not parent_path.startswith("/") and \ - not name.startswith("/"): - file_dir = posixpath.dirname(parent_path) - name = posixpath.normpath(posixpath.join(file_dir, name)) - return name - - def _create_template(self, name): - return Template(self.dict[name], name=name, loader=self) - - -class _Node(object): - def each_child(self): - return () - - def generate(self, writer): - raise NotImplementedError() - - def find_named_blocks(self, loader, named_blocks): - for child in self.each_child(): - child.find_named_blocks(loader, named_blocks) - - -class _File(_Node): - def __init__(self, template, body): - self.template = template - self.body = body - self.line = 0 - - def generate(self, writer): - writer.write_line("def _tt_execute():", self.line) - with writer.indent(): - writer.write_line("_tt_buffer = []", self.line) - writer.write_line("_tt_append = _tt_buffer.append", self.line) - self.body.generate(writer) - writer.write_line("return _tt_utf8('').join(_tt_buffer)", self.line) - - def each_child(self): - return (self.body,) - - -class _ChunkList(_Node): - def __init__(self, chunks): - self.chunks = chunks - - def generate(self, writer): - for chunk in self.chunks: - chunk.generate(writer) - - def each_child(self): - return self.chunks - - -class _NamedBlock(_Node): - def __init__(self, name, body, template, line): - self.name = name - self.body = body - self.template = template - self.line = line - - def each_child(self): - return (self.body,) - - def generate(self, writer): - block = writer.named_blocks[self.name] - with writer.include(block.template, self.line): - block.body.generate(writer) - - def find_named_blocks(self, loader, named_blocks): - named_blocks[self.name] = self - _Node.find_named_blocks(self, loader, named_blocks) - - -class _ExtendsBlock(_Node): - def __init__(self, name): - self.name = name - - -class _IncludeBlock(_Node): - def __init__(self, name, reader, line): - self.name = name - self.template_name = reader.name - self.line = line - - def find_named_blocks(self, loader, named_blocks): - included = loader.load(self.name, self.template_name) - included.file.find_named_blocks(loader, named_blocks) - - def generate(self, writer): - included = writer.loader.load(self.name, self.template_name) - with writer.include(included, self.line): - included.file.body.generate(writer) - - -class _ApplyBlock(_Node): - def __init__(self, method, line, body=None): - self.method = method - self.line = line - self.body = body - - def each_child(self): - return (self.body,) - - def generate(self, writer): - method_name = "_tt_apply%d" % writer.apply_counter - writer.apply_counter += 1 - writer.write_line("def %s():" % method_name, self.line) - with writer.indent(): - writer.write_line("_tt_buffer = []", self.line) - writer.write_line("_tt_append = _tt_buffer.append", self.line) - self.body.generate(writer) - writer.write_line("return _tt_utf8('').join(_tt_buffer)", self.line) - writer.write_line("_tt_append(_tt_utf8(%s(%s())))" % ( - self.method, method_name), self.line) - - -class _ControlBlock(_Node): - def __init__(self, statement, line, body=None): - self.statement = statement - self.line = line - self.body = body - - def each_child(self): - return (self.body,) - - def generate(self, writer): - writer.write_line("%s:" % self.statement, self.line) - with writer.indent(): - self.body.generate(writer) - # Just in case the body was empty - writer.write_line("pass", self.line) - - -class _IntermediateControlBlock(_Node): - def __init__(self, statement, line): - self.statement = statement - self.line = line - - def generate(self, writer): - # In case the previous block was empty - writer.write_line("pass", self.line) - writer.write_line("%s:" % self.statement, self.line, writer.indent_size() - 1) - - -class _Statement(_Node): - def __init__(self, statement, line): - self.statement = statement - self.line = line - - def generate(self, writer): - writer.write_line(self.statement, self.line) - - -class _Expression(_Node): - def __init__(self, expression, line, raw=False): - self.expression = expression - self.line = line - self.raw = raw - - def generate(self, writer): - writer.write_line("_tt_tmp = %s" % self.expression, self.line) - writer.write_line("if isinstance(_tt_tmp, _tt_string_types):" - " _tt_tmp = _tt_utf8(_tt_tmp)", self.line) - writer.write_line("else: _tt_tmp = _tt_utf8(str(_tt_tmp))", self.line) - if not self.raw and writer.current_template.autoescape is not None: - # In python3 functions like xhtml_escape return unicode, - # so we have to convert to utf8 again. - writer.write_line("_tt_tmp = _tt_utf8(%s(_tt_tmp))" % - writer.current_template.autoescape, self.line) - writer.write_line("_tt_append(_tt_tmp)", self.line) - - -class _Module(_Expression): - def __init__(self, expression, line): - super(_Module, self).__init__("_tt_modules." + expression, line, - raw=True) - - -class _Text(_Node): - def __init__(self, value, line, whitespace): - self.value = value - self.line = line - self.whitespace = whitespace - - def generate(self, writer): - value = self.value - - # Compress whitespace if requested, with a crude heuristic to avoid - # altering preformatted whitespace. - if "
    " not in value:
-            value = filter_whitespace(self.whitespace, value)
-
-        if value:
-            writer.write_line('_tt_append(%r)' % escape.utf8(value), self.line)
-
-
-class ParseError(Exception):
-    """Raised for template syntax errors.
-
-    ``ParseError`` instances have ``filename`` and ``lineno`` attributes
-    indicating the position of the error.
-
-    .. versionchanged:: 4.3
-       Added ``filename`` and ``lineno`` attributes.
-    """
-    def __init__(self, message, filename=None, lineno=0):
-        self.message = message
-        # The names "filename" and "lineno" are chosen for consistency
-        # with python SyntaxError.
-        self.filename = filename
-        self.lineno = lineno
-
-    def __str__(self):
-        return '%s at %s:%d' % (self.message, self.filename, self.lineno)
-
-
-class _CodeWriter(object):
-    def __init__(self, file, named_blocks, loader, current_template):
-        self.file = file
-        self.named_blocks = named_blocks
-        self.loader = loader
-        self.current_template = current_template
-        self.apply_counter = 0
-        self.include_stack = []
-        self._indent = 0
-
-    def indent_size(self):
-        return self._indent
-
-    def indent(self):
-        class Indenter(object):
-            def __enter__(_):
-                self._indent += 1
-                return self
-
-            def __exit__(_, *args):
-                assert self._indent > 0
-                self._indent -= 1
-
-        return Indenter()
-
-    def include(self, template, line):
-        self.include_stack.append((self.current_template, line))
-        self.current_template = template
-
-        class IncludeTemplate(object):
-            def __enter__(_):
-                return self
-
-            def __exit__(_, *args):
-                self.current_template = self.include_stack.pop()[0]
-
-        return IncludeTemplate()
-
-    def write_line(self, line, line_number, indent=None):
-        if indent is None:
-            indent = self._indent
-        line_comment = '  # %s:%d' % (self.current_template.name, line_number)
-        if self.include_stack:
-            ancestors = ["%s:%d" % (tmpl.name, lineno)
-                         for (tmpl, lineno) in self.include_stack]
-            line_comment += ' (via %s)' % ', '.join(reversed(ancestors))
-        print("    " * indent + line + line_comment, file=self.file)
-
-
-class _TemplateReader(object):
-    def __init__(self, name, text, whitespace):
-        self.name = name
-        self.text = text
-        self.whitespace = whitespace
-        self.line = 1
-        self.pos = 0
-
-    def find(self, needle, start=0, end=None):
-        assert start >= 0, start
-        pos = self.pos
-        start += pos
-        if end is None:
-            index = self.text.find(needle, start)
-        else:
-            end += pos
-            assert end >= start
-            index = self.text.find(needle, start, end)
-        if index != -1:
-            index -= pos
-        return index
-
-    def consume(self, count=None):
-        if count is None:
-            count = len(self.text) - self.pos
-        newpos = self.pos + count
-        self.line += self.text.count("\n", self.pos, newpos)
-        s = self.text[self.pos:newpos]
-        self.pos = newpos
-        return s
-
-    def remaining(self):
-        return len(self.text) - self.pos
-
-    def __len__(self):
-        return self.remaining()
-
-    def __getitem__(self, key):
-        if type(key) is slice:
-            size = len(self)
-            start, stop, step = key.indices(size)
-            if start is None:
-                start = self.pos
-            else:
-                start += self.pos
-            if stop is not None:
-                stop += self.pos
-            return self.text[slice(start, stop, step)]
-        elif key < 0:
-            return self.text[key]
-        else:
-            return self.text[self.pos + key]
-
-    def __str__(self):
-        return self.text[self.pos:]
-
-    def raise_parse_error(self, msg):
-        raise ParseError(msg, self.name, self.line)
-
-
-def _format_code(code):
-    lines = code.splitlines()
-    format = "%%%dd  %%s\n" % len(repr(len(lines) + 1))
-    return "".join([format % (i + 1, line) for (i, line) in enumerate(lines)])
-
-
-def _parse(reader, template, in_block=None, in_loop=None):
-    body = _ChunkList([])
-    while True:
-        # Find next template directive
-        curly = 0
-        while True:
-            curly = reader.find("{", curly)
-            if curly == -1 or curly + 1 == reader.remaining():
-                # EOF
-                if in_block:
-                    reader.raise_parse_error(
-                        "Missing {%% end %%} block for %s" % in_block)
-                body.chunks.append(_Text(reader.consume(), reader.line,
-                                         reader.whitespace))
-                return body
-            # If the first curly brace is not the start of a special token,
-            # start searching from the character after it
-            if reader[curly + 1] not in ("{", "%", "#"):
-                curly += 1
-                continue
-            # When there are more than 2 curlies in a row, use the
-            # innermost ones.  This is useful when generating languages
-            # like latex where curlies are also meaningful
-            if (curly + 2 < reader.remaining() and
-                    reader[curly + 1] == '{' and reader[curly + 2] == '{'):
-                curly += 1
-                continue
-            break
-
-        # Append any text before the special token
-        if curly > 0:
-            cons = reader.consume(curly)
-            body.chunks.append(_Text(cons, reader.line,
-                                     reader.whitespace))
-
-        start_brace = reader.consume(2)
-        line = reader.line
-
-        # Template directives may be escaped as "{{!" or "{%!".
-        # In this case output the braces and consume the "!".
-        # This is especially useful in conjunction with jquery templates,
-        # which also use double braces.
-        if reader.remaining() and reader[0] == "!":
-            reader.consume(1)
-            body.chunks.append(_Text(start_brace, line,
-                                     reader.whitespace))
-            continue
-
-        # Comment
-        if start_brace == "{#":
-            end = reader.find("#}")
-            if end == -1:
-                reader.raise_parse_error("Missing end comment #}")
-            contents = reader.consume(end).strip()
-            reader.consume(2)
-            continue
-
-        # Expression
-        if start_brace == "{{":
-            end = reader.find("}}")
-            if end == -1:
-                reader.raise_parse_error("Missing end expression }}")
-            contents = reader.consume(end).strip()
-            reader.consume(2)
-            if not contents:
-                reader.raise_parse_error("Empty expression")
-            body.chunks.append(_Expression(contents, line))
-            continue
-
-        # Block
-        assert start_brace == "{%", start_brace
-        end = reader.find("%}")
-        if end == -1:
-            reader.raise_parse_error("Missing end block %}")
-        contents = reader.consume(end).strip()
-        reader.consume(2)
-        if not contents:
-            reader.raise_parse_error("Empty block tag ({% %})")
-
-        operator, space, suffix = contents.partition(" ")
-        suffix = suffix.strip()
-
-        # Intermediate ("else", "elif", etc) blocks
-        intermediate_blocks = {
-            "else": set(["if", "for", "while", "try"]),
-            "elif": set(["if"]),
-            "except": set(["try"]),
-            "finally": set(["try"]),
-        }
-        allowed_parents = intermediate_blocks.get(operator)
-        if allowed_parents is not None:
-            if not in_block:
-                reader.raise_parse_error("%s outside %s block" %
-                                         (operator, allowed_parents))
-            if in_block not in allowed_parents:
-                reader.raise_parse_error(
-                    "%s block cannot be attached to %s block" %
-                    (operator, in_block))
-            body.chunks.append(_IntermediateControlBlock(contents, line))
-            continue
-
-        # End tag
-        elif operator == "end":
-            if not in_block:
-                reader.raise_parse_error("Extra {% end %} block")
-            return body
-
-        elif operator in ("extends", "include", "set", "import", "from",
-                          "comment", "autoescape", "whitespace", "raw",
-                          "module"):
-            if operator == "comment":
-                continue
-            if operator == "extends":
-                suffix = suffix.strip('"').strip("'")
-                if not suffix:
-                    reader.raise_parse_error("extends missing file path")
-                block = _ExtendsBlock(suffix)
-            elif operator in ("import", "from"):
-                if not suffix:
-                    reader.raise_parse_error("import missing statement")
-                block = _Statement(contents, line)
-            elif operator == "include":
-                suffix = suffix.strip('"').strip("'")
-                if not suffix:
-                    reader.raise_parse_error("include missing file path")
-                block = _IncludeBlock(suffix, reader, line)
-            elif operator == "set":
-                if not suffix:
-                    reader.raise_parse_error("set missing statement")
-                block = _Statement(suffix, line)
-            elif operator == "autoescape":
-                fn = suffix.strip()
-                if fn == "None":
-                    fn = None
-                template.autoescape = fn
-                continue
-            elif operator == "whitespace":
-                mode = suffix.strip()
-                # Validate the selected mode
-                filter_whitespace(mode, '')
-                reader.whitespace = mode
-                continue
-            elif operator == "raw":
-                block = _Expression(suffix, line, raw=True)
-            elif operator == "module":
-                block = _Module(suffix, line)
-            body.chunks.append(block)
-            continue
-
-        elif operator in ("apply", "block", "try", "if", "for", "while"):
-            # parse inner body recursively
-            if operator in ("for", "while"):
-                block_body = _parse(reader, template, operator, operator)
-            elif operator == "apply":
-                # apply creates a nested function so syntactically it's not
-                # in the loop.
-                block_body = _parse(reader, template, operator, None)
-            else:
-                block_body = _parse(reader, template, operator, in_loop)
-
-            if operator == "apply":
-                if not suffix:
-                    reader.raise_parse_error("apply missing method name")
-                block = _ApplyBlock(suffix, line, block_body)
-            elif operator == "block":
-                if not suffix:
-                    reader.raise_parse_error("block missing name")
-                block = _NamedBlock(suffix, block_body, template, line)
-            else:
-                block = _ControlBlock(contents, line, block_body)
-            body.chunks.append(block)
-            continue
-
-        elif operator in ("break", "continue"):
-            if not in_loop:
-                reader.raise_parse_error("%s outside %s block" %
-                                         (operator, set(["for", "while"])))
-            body.chunks.append(_Statement(contents, line))
-            continue
-
-        else:
-            reader.raise_parse_error("unknown operator: %r" % operator)
diff --git a/salt/ext/tornado/test/__init__.py b/salt/ext/tornado/test/__init__.py
deleted file mode 100644
index 388083ed935..00000000000
--- a/salt/ext/tornado/test/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-# pylint: skip-file
diff --git a/salt/ext/tornado/test/__main__.py b/salt/ext/tornado/test/__main__.py
deleted file mode 100644
index cf96e5932de..00000000000
--- a/salt/ext/tornado/test/__main__.py
+++ /dev/null
@@ -1,15 +0,0 @@
-"""Shim to allow python -m tornado.test.
-
-This only works in python 2.7+.
-"""
-# pylint: skip-file
-from __future__ import absolute_import, division, print_function
-
-from salt.ext.tornado.test.runtests import all, main
-
-# tornado.testing.main autodiscovery relies on 'all' being present in
-# the main module, so import it here even though it is not used directly.
-# The following line prevents a pyflakes warning.
-all = all
-
-main()
diff --git a/salt/ext/tornado/test/asyncio_test.py b/salt/ext/tornado/test/asyncio_test.py
deleted file mode 100644
index 32746d2150e..00000000000
--- a/salt/ext/tornado/test/asyncio_test.py
+++ /dev/null
@@ -1,121 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-# pylint: skip-file
-
-from __future__ import absolute_import, division, print_function
-
-from salt.ext.tornado import gen
-from salt.ext.tornado.testing import AsyncTestCase, gen_test
-from salt.ext.tornado.test.util import unittest, skipBefore33, skipBefore35, exec_test
-
-try:
-    from salt.ext.tornado.platform.asyncio import asyncio
-except ImportError:
-    asyncio = None
-else:
-    from salt.ext.tornado.platform.asyncio import AsyncIOLoop, to_asyncio_future
-    # This is used in dynamically-evaluated code, so silence pyflakes.
-    to_asyncio_future
-
-
-@unittest.skipIf(asyncio is None, "asyncio module not present")
-class AsyncIOLoopTest(AsyncTestCase):
-    def get_new_ioloop(self):
-        io_loop = AsyncIOLoop()
-        asyncio.set_event_loop(io_loop.asyncio_loop)
-        return io_loop
-
-    def test_asyncio_callback(self):
-        # Basic test that the asyncio loop is set up correctly.
-        asyncio.get_event_loop().call_soon(self.stop)
-        self.wait()
-
-    @gen_test
-    def test_asyncio_future(self):
-        # Test that we can yield an asyncio future from a tornado coroutine.
-        # Without 'yield from', we must wrap coroutines in ensure_future,
-        # which was introduced during Python 3.4, deprecating the prior "async".
-        if hasattr(asyncio, 'ensure_future'):
-            ensure_future = asyncio.ensure_future
-        else:
-            # async is a reserved word in Python 3.7
-            ensure_future = getattr(asyncio, "async")
-
-        x = yield ensure_future(
-            asyncio.get_event_loop().run_in_executor(None, lambda: 42))
-        self.assertEqual(x, 42)
-
-    @skipBefore33
-    @gen_test
-    def test_asyncio_yield_from(self):
-        # Test that we can use asyncio coroutines with 'yield from'
-        # instead of asyncio.async(). This requires python 3.3 syntax.
-        namespace = exec_test(globals(), locals(), """
-        @gen.coroutine
-        def f():
-            event_loop = asyncio.get_event_loop()
-            x = yield from event_loop.run_in_executor(None, lambda: 42)
-            return x
-        """)
-        result = yield namespace['f']()
-        self.assertEqual(result, 42)
-
-    @skipBefore35
-    def test_asyncio_adapter(self):
-        # This test demonstrates that when using the asyncio coroutine
-        # runner (i.e. run_until_complete), the to_asyncio_future
-        # adapter is needed. No adapter is needed in the other direction,
-        # as demonstrated by other tests in the package.
-        @gen.coroutine
-        def tornado_coroutine():
-            yield gen.Task(self.io_loop.add_callback)
-            raise gen.Return(42)
-        native_coroutine_without_adapter = exec_test(globals(), locals(), """
-        async def native_coroutine_without_adapter():
-            return await tornado_coroutine()
-        """)["native_coroutine_without_adapter"]
-
-        native_coroutine_with_adapter = exec_test(globals(), locals(), """
-        async def native_coroutine_with_adapter():
-            return await to_asyncio_future(tornado_coroutine())
-        """)["native_coroutine_with_adapter"]
-
-        # Use the adapter, but two degrees from the tornado coroutine.
-        native_coroutine_with_adapter2 = exec_test(globals(), locals(), """
-        async def native_coroutine_with_adapter2():
-            return await to_asyncio_future(native_coroutine_without_adapter())
-        """)["native_coroutine_with_adapter2"]
-
-        # Tornado supports native coroutines both with and without adapters
-        self.assertEqual(
-            self.io_loop.run_sync(native_coroutine_without_adapter),
-            42)
-        self.assertEqual(
-            self.io_loop.run_sync(native_coroutine_with_adapter),
-            42)
-        self.assertEqual(
-            self.io_loop.run_sync(native_coroutine_with_adapter2),
-            42)
-
-        # Asyncio only supports coroutines that yield asyncio-compatible
-        # Futures.
-        with self.assertRaises(RuntimeError):
-            asyncio.get_event_loop().run_until_complete(
-                native_coroutine_without_adapter())
-        self.assertEqual(
-            asyncio.get_event_loop().run_until_complete(
-                native_coroutine_with_adapter()),
-            42)
-        self.assertEqual(
-            asyncio.get_event_loop().run_until_complete(
-                native_coroutine_with_adapter2()),
-            42)
diff --git a/salt/ext/tornado/test/auth_test.py b/salt/ext/tornado/test/auth_test.py
deleted file mode 100644
index 38196203d65..00000000000
--- a/salt/ext/tornado/test/auth_test.py
+++ /dev/null
@@ -1,548 +0,0 @@
-# These tests do not currently do much to verify the correct implementation
-# of the openid/oauth protocols, they just exercise the major code paths
-# and ensure that it doesn't blow up (e.g. with unicode/bytes issues in
-# python 3)
-# pylint: skip-file
-
-
-from __future__ import absolute_import, division, print_function
-from salt.ext.tornado.auth import OpenIdMixin, OAuthMixin, OAuth2Mixin, TwitterMixin, AuthError, GoogleOAuth2Mixin, FacebookGraphMixin
-from salt.ext.tornado.concurrent import Future
-from salt.ext.tornado.escape import json_decode
-from salt.ext.tornado import gen
-from salt.ext.tornado.httputil import url_concat
-from salt.ext.tornado.log import gen_log
-from salt.ext.tornado.testing import AsyncHTTPTestCase, ExpectLog
-from salt.ext.tornado.web import RequestHandler, Application, asynchronous, HTTPError
-
-
-class OpenIdClientLoginHandler(RequestHandler, OpenIdMixin):
-    def initialize(self, test):
-        self._OPENID_ENDPOINT = test.get_url('/openid/server/authenticate')
-
-    @asynchronous
-    def get(self):
-        if self.get_argument('openid.mode', None):
-            self.get_authenticated_user(
-                self.on_user, http_client=self.settings['http_client'])
-            return
-        res = self.authenticate_redirect()
-        assert isinstance(res, Future)
-        assert res.done()
-
-    def on_user(self, user):
-        if user is None:
-            raise Exception("user is None")
-        self.finish(user)
-
-
-class OpenIdServerAuthenticateHandler(RequestHandler):
-    def post(self):
-        if self.get_argument('openid.mode') != 'check_authentication':
-            raise Exception("incorrect openid.mode %r")
-        self.write('is_valid:true')
-
-
-class OAuth1ClientLoginHandler(RequestHandler, OAuthMixin):
-    def initialize(self, test, version):
-        self._OAUTH_VERSION = version
-        self._OAUTH_REQUEST_TOKEN_URL = test.get_url('/oauth1/server/request_token')
-        self._OAUTH_AUTHORIZE_URL = test.get_url('/oauth1/server/authorize')
-        self._OAUTH_ACCESS_TOKEN_URL = test.get_url('/oauth1/server/access_token')
-
-    def _oauth_consumer_token(self):
-        return dict(key='asdf', secret='qwer')
-
-    @asynchronous
-    def get(self):
-        if self.get_argument('oauth_token', None):
-            self.get_authenticated_user(
-                self.on_user, http_client=self.settings['http_client'])
-            return
-        res = self.authorize_redirect(http_client=self.settings['http_client'])
-        assert isinstance(res, Future)
-
-    def on_user(self, user):
-        if user is None:
-            raise Exception("user is None")
-        self.finish(user)
-
-    def _oauth_get_user(self, access_token, callback):
-        if self.get_argument('fail_in_get_user', None):
-            raise Exception("failing in get_user")
-        if access_token != dict(key='uiop', secret='5678'):
-            raise Exception("incorrect access token %r" % access_token)
-        callback(dict(email='foo@example.com'))
-
-
-class OAuth1ClientLoginCoroutineHandler(OAuth1ClientLoginHandler):
-    """Replaces OAuth1ClientLoginCoroutineHandler's get() with a coroutine."""
-    @gen.coroutine
-    def get(self):
-        if self.get_argument('oauth_token', None):
-            # Ensure that any exceptions are set on the returned Future,
-            # not simply thrown into the surrounding StackContext.
-            try:
-                yield self.get_authenticated_user()
-            except Exception as e:
-                self.set_status(503)
-                self.write("got exception: %s" % e)
-        else:
-            yield self.authorize_redirect()
-
-
-class OAuth1ClientRequestParametersHandler(RequestHandler, OAuthMixin):
-    def initialize(self, version):
-        self._OAUTH_VERSION = version
-
-    def _oauth_consumer_token(self):
-        return dict(key='asdf', secret='qwer')
-
-    def get(self):
-        params = self._oauth_request_parameters(
-            'http://www.example.com/api/asdf',
-            dict(key='uiop', secret='5678'),
-            parameters=dict(foo='bar'))
-        self.write(params)
-
-
-class OAuth1ServerRequestTokenHandler(RequestHandler):
-    def get(self):
-        self.write('oauth_token=zxcv&oauth_token_secret=1234')
-
-
-class OAuth1ServerAccessTokenHandler(RequestHandler):
-    def get(self):
-        self.write('oauth_token=uiop&oauth_token_secret=5678')
-
-
-class OAuth2ClientLoginHandler(RequestHandler, OAuth2Mixin):
-    def initialize(self, test):
-        self._OAUTH_AUTHORIZE_URL = test.get_url('/oauth2/server/authorize')
-
-    def get(self):
-        res = self.authorize_redirect()
-        assert isinstance(res, Future)
-        assert res.done()
-
-
-class FacebookClientLoginHandler(RequestHandler, FacebookGraphMixin):
-    def initialize(self, test):
-        self._OAUTH_AUTHORIZE_URL = test.get_url('/facebook/server/authorize')
-        self._OAUTH_ACCESS_TOKEN_URL = test.get_url('/facebook/server/access_token')
-        self._FACEBOOK_BASE_URL = test.get_url('/facebook/server')
-
-    @gen.coroutine
-    def get(self):
-        if self.get_argument("code", None):
-            user = yield self.get_authenticated_user(
-                redirect_uri=self.request.full_url(),
-                client_id=self.settings["facebook_api_key"],
-                client_secret=self.settings["facebook_secret"],
-                code=self.get_argument("code"))
-            self.write(user)
-        else:
-            yield self.authorize_redirect(
-                redirect_uri=self.request.full_url(),
-                client_id=self.settings["facebook_api_key"],
-                extra_params={"scope": "read_stream,offline_access"})
-
-
-class FacebookServerAccessTokenHandler(RequestHandler):
-    def get(self):
-        self.write(dict(access_token="asdf", expires_in=3600))
-
-
-class FacebookServerMeHandler(RequestHandler):
-    def get(self):
-        self.write('{}')
-
-
-class TwitterClientHandler(RequestHandler, TwitterMixin):
-    def initialize(self, test):
-        self._OAUTH_REQUEST_TOKEN_URL = test.get_url('/oauth1/server/request_token')
-        self._OAUTH_ACCESS_TOKEN_URL = test.get_url('/twitter/server/access_token')
-        self._OAUTH_AUTHORIZE_URL = test.get_url('/oauth1/server/authorize')
-        self._TWITTER_BASE_URL = test.get_url('/twitter/api')
-
-    def get_auth_http_client(self):
-        return self.settings['http_client']
-
-
-class TwitterClientLoginHandler(TwitterClientHandler):
-    @asynchronous
-    def get(self):
-        if self.get_argument("oauth_token", None):
-            self.get_authenticated_user(self.on_user)
-            return
-        self.authorize_redirect()
-
-    def on_user(self, user):
-        if user is None:
-            raise Exception("user is None")
-        self.finish(user)
-
-
-class TwitterClientLoginGenEngineHandler(TwitterClientHandler):
-    @asynchronous
-    @gen.engine
-    def get(self):
-        if self.get_argument("oauth_token", None):
-            user = yield self.get_authenticated_user()
-            self.finish(user)
-        else:
-            # Old style: with @gen.engine we can ignore the Future from
-            # authorize_redirect.
-            self.authorize_redirect()
-
-
-class TwitterClientLoginGenCoroutineHandler(TwitterClientHandler):
-    @gen.coroutine
-    def get(self):
-        if self.get_argument("oauth_token", None):
-            user = yield self.get_authenticated_user()
-            self.finish(user)
-        else:
-            # New style: with @gen.coroutine the result must be yielded
-            # or else the request will be auto-finished too soon.
-            yield self.authorize_redirect()
-
-
-class TwitterClientShowUserHandler(TwitterClientHandler):
-    @asynchronous
-    @gen.engine
-    def get(self):
-        # TODO: would be nice to go through the login flow instead of
-        # cheating with a hard-coded access token.
-        response = yield gen.Task(self.twitter_request,
-                                  '/users/show/%s' % self.get_argument('name'),
-                                  access_token=dict(key='hjkl', secret='vbnm'))
-        if response is None:
-            self.set_status(500)
-            self.finish('error from twitter request')
-        else:
-            self.finish(response)
-
-
-class TwitterClientShowUserFutureHandler(TwitterClientHandler):
-    @asynchronous
-    @gen.engine
-    def get(self):
-        try:
-            response = yield self.twitter_request(
-                '/users/show/%s' % self.get_argument('name'),
-                access_token=dict(key='hjkl', secret='vbnm'))
-        except AuthError as e:
-            self.set_status(500)
-            self.finish(str(e))
-            return
-        assert response is not None
-        self.finish(response)
-
-
-class TwitterServerAccessTokenHandler(RequestHandler):
-    def get(self):
-        self.write('oauth_token=hjkl&oauth_token_secret=vbnm&screen_name=foo')
-
-
-class TwitterServerShowUserHandler(RequestHandler):
-    def get(self, screen_name):
-        if screen_name == 'error':
-            raise HTTPError(500)
-        assert 'oauth_nonce' in self.request.arguments
-        assert 'oauth_timestamp' in self.request.arguments
-        assert 'oauth_signature' in self.request.arguments
-        assert self.get_argument('oauth_consumer_key') == 'test_twitter_consumer_key'
-        assert self.get_argument('oauth_signature_method') == 'HMAC-SHA1'
-        assert self.get_argument('oauth_version') == '1.0'
-        assert self.get_argument('oauth_token') == 'hjkl'
-        self.write(dict(screen_name=screen_name, name=screen_name.capitalize()))
-
-
-class TwitterServerVerifyCredentialsHandler(RequestHandler):
-    def get(self):
-        assert 'oauth_nonce' in self.request.arguments
-        assert 'oauth_timestamp' in self.request.arguments
-        assert 'oauth_signature' in self.request.arguments
-        assert self.get_argument('oauth_consumer_key') == 'test_twitter_consumer_key'
-        assert self.get_argument('oauth_signature_method') == 'HMAC-SHA1'
-        assert self.get_argument('oauth_version') == '1.0'
-        assert self.get_argument('oauth_token') == 'hjkl'
-        self.write(dict(screen_name='foo', name='Foo'))
-
-
-class AuthTest(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application(
-            [
-                # test endpoints
-                ('/openid/client/login', OpenIdClientLoginHandler, dict(test=self)),
-                ('/oauth10/client/login', OAuth1ClientLoginHandler,
-                 dict(test=self, version='1.0')),
-                ('/oauth10/client/request_params',
-                 OAuth1ClientRequestParametersHandler,
-                 dict(version='1.0')),
-                ('/oauth10a/client/login', OAuth1ClientLoginHandler,
-                 dict(test=self, version='1.0a')),
-                ('/oauth10a/client/login_coroutine',
-                 OAuth1ClientLoginCoroutineHandler,
-                 dict(test=self, version='1.0a')),
-                ('/oauth10a/client/request_params',
-                 OAuth1ClientRequestParametersHandler,
-                 dict(version='1.0a')),
-                ('/oauth2/client/login', OAuth2ClientLoginHandler, dict(test=self)),
-
-                ('/facebook/client/login', FacebookClientLoginHandler, dict(test=self)),
-
-                ('/twitter/client/login', TwitterClientLoginHandler, dict(test=self)),
-                ('/twitter/client/login_gen_engine', TwitterClientLoginGenEngineHandler, dict(test=self)),
-                ('/twitter/client/login_gen_coroutine', TwitterClientLoginGenCoroutineHandler, dict(test=self)),
-                ('/twitter/client/show_user', TwitterClientShowUserHandler, dict(test=self)),
-                ('/twitter/client/show_user_future', TwitterClientShowUserFutureHandler, dict(test=self)),
-
-                # simulated servers
-                ('/openid/server/authenticate', OpenIdServerAuthenticateHandler),
-                ('/oauth1/server/request_token', OAuth1ServerRequestTokenHandler),
-                ('/oauth1/server/access_token', OAuth1ServerAccessTokenHandler),
-
-                ('/facebook/server/access_token', FacebookServerAccessTokenHandler),
-                ('/facebook/server/me', FacebookServerMeHandler),
-                ('/twitter/server/access_token', TwitterServerAccessTokenHandler),
-                (r'/twitter/api/users/show/(.*)\.json', TwitterServerShowUserHandler),
-                (r'/twitter/api/account/verify_credentials\.json', TwitterServerVerifyCredentialsHandler),
-            ],
-            http_client=self.http_client,
-            twitter_consumer_key='test_twitter_consumer_key',
-            twitter_consumer_secret='test_twitter_consumer_secret',
-            facebook_api_key='test_facebook_api_key',
-            facebook_secret='test_facebook_secret')
-
-    def test_openid_redirect(self):
-        response = self.fetch('/openid/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue(
-            '/openid/server/authenticate?' in response.headers['Location'])
-
-    def test_openid_get_user(self):
-        response = self.fetch('/openid/client/login?openid.mode=blah&openid.ns.ax=http://openid.net/srv/ax/1.0&openid.ax.type.email=http://axschema.org/contact/email&openid.ax.value.email=foo@example.com')
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed["email"], "foo@example.com")
-
-    def test_oauth10_redirect(self):
-        response = self.fetch('/oauth10/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue(response.headers['Location'].endswith(
-            '/oauth1/server/authorize?oauth_token=zxcv'))
-        # the cookie is base64('zxcv')|base64('1234')
-        self.assertTrue(
-            '_oauth_request_token="enhjdg==|MTIzNA=="' in response.headers['Set-Cookie'],
-            response.headers['Set-Cookie'])
-
-    def test_oauth10_get_user(self):
-        response = self.fetch(
-            '/oauth10/client/login?oauth_token=zxcv',
-            headers={'Cookie': '_oauth_request_token=enhjdg==|MTIzNA=='})
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed['email'], 'foo@example.com')
-        self.assertEqual(parsed['access_token'], dict(key='uiop', secret='5678'))
-
-    def test_oauth10_request_parameters(self):
-        response = self.fetch('/oauth10/client/request_params')
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed['oauth_consumer_key'], 'asdf')
-        self.assertEqual(parsed['oauth_token'], 'uiop')
-        self.assertTrue('oauth_nonce' in parsed)
-        self.assertTrue('oauth_signature' in parsed)
-
-    def test_oauth10a_redirect(self):
-        response = self.fetch('/oauth10a/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue(response.headers['Location'].endswith(
-            '/oauth1/server/authorize?oauth_token=zxcv'))
-        # the cookie is base64('zxcv')|base64('1234')
-        self.assertTrue(
-            '_oauth_request_token="enhjdg==|MTIzNA=="' in response.headers['Set-Cookie'],
-            response.headers['Set-Cookie'])
-
-    def test_oauth10a_get_user(self):
-        response = self.fetch(
-            '/oauth10a/client/login?oauth_token=zxcv',
-            headers={'Cookie': '_oauth_request_token=enhjdg==|MTIzNA=='})
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed['email'], 'foo@example.com')
-        self.assertEqual(parsed['access_token'], dict(key='uiop', secret='5678'))
-
-    def test_oauth10a_request_parameters(self):
-        response = self.fetch('/oauth10a/client/request_params')
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed['oauth_consumer_key'], 'asdf')
-        self.assertEqual(parsed['oauth_token'], 'uiop')
-        self.assertTrue('oauth_nonce' in parsed)
-        self.assertTrue('oauth_signature' in parsed)
-
-    def test_oauth10a_get_user_coroutine_exception(self):
-        response = self.fetch(
-            '/oauth10a/client/login_coroutine?oauth_token=zxcv&fail_in_get_user=true',
-            headers={'Cookie': '_oauth_request_token=enhjdg==|MTIzNA=='})
-        self.assertEqual(response.code, 503)
-
-    def test_oauth2_redirect(self):
-        response = self.fetch('/oauth2/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue('/oauth2/server/authorize?' in response.headers['Location'])
-
-    def test_facebook_login(self):
-        response = self.fetch('/facebook/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue('/facebook/server/authorize?' in response.headers['Location'])
-        response = self.fetch('/facebook/client/login?code=1234', follow_redirects=False)
-        self.assertEqual(response.code, 200)
-        user = json_decode(response.body)
-        self.assertEqual(user['access_token'], 'asdf')
-        self.assertEqual(user['session_expires'], '3600')
-
-    def base_twitter_redirect(self, url):
-        # Same as test_oauth10a_redirect
-        response = self.fetch(url, follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue(response.headers['Location'].endswith(
-            '/oauth1/server/authorize?oauth_token=zxcv'))
-        # the cookie is base64('zxcv')|base64('1234')
-        self.assertTrue(
-            '_oauth_request_token="enhjdg==|MTIzNA=="' in response.headers['Set-Cookie'],
-            response.headers['Set-Cookie'])
-
-    def test_twitter_redirect(self):
-        self.base_twitter_redirect('/twitter/client/login')
-
-    def test_twitter_redirect_gen_engine(self):
-        self.base_twitter_redirect('/twitter/client/login_gen_engine')
-
-    def test_twitter_redirect_gen_coroutine(self):
-        self.base_twitter_redirect('/twitter/client/login_gen_coroutine')
-
-    def test_twitter_get_user(self):
-        response = self.fetch(
-            '/twitter/client/login?oauth_token=zxcv',
-            headers={'Cookie': '_oauth_request_token=enhjdg==|MTIzNA=='})
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed,
-                         {u'access_token': {u'key': u'hjkl',
-                                            u'screen_name': u'foo',
-                                            u'secret': u'vbnm'},
-                          u'name': u'Foo',
-                          u'screen_name': u'foo',
-                          u'username': u'foo'})
-
-    def test_twitter_show_user(self):
-        response = self.fetch('/twitter/client/show_user?name=somebody')
-        response.rethrow()
-        self.assertEqual(json_decode(response.body),
-                         {'name': 'Somebody', 'screen_name': 'somebody'})
-
-    def test_twitter_show_user_error(self):
-        with ExpectLog(gen_log, 'Error response HTTP 500'):
-            response = self.fetch('/twitter/client/show_user?name=error')
-        self.assertEqual(response.code, 500)
-        self.assertEqual(response.body, b'error from twitter request')
-
-    def test_twitter_show_user_future(self):
-        response = self.fetch('/twitter/client/show_user_future?name=somebody')
-        response.rethrow()
-        self.assertEqual(json_decode(response.body),
-                         {'name': 'Somebody', 'screen_name': 'somebody'})
-
-    def test_twitter_show_user_future_error(self):
-        response = self.fetch('/twitter/client/show_user_future?name=error')
-        self.assertEqual(response.code, 500)
-        self.assertIn(b'Error response HTTP 500', response.body)
-
-
-class GoogleLoginHandler(RequestHandler, GoogleOAuth2Mixin):
-    def initialize(self, test):
-        self.test = test
-        self._OAUTH_REDIRECT_URI = test.get_url('/client/login')
-        self._OAUTH_AUTHORIZE_URL = test.get_url('/google/oauth2/authorize')
-        self._OAUTH_ACCESS_TOKEN_URL = test.get_url('/google/oauth2/token')
-
-    @gen.coroutine
-    def get(self):
-        code = self.get_argument('code', None)
-        if code is not None:
-            # retrieve authenticate google user
-            access = yield self.get_authenticated_user(self._OAUTH_REDIRECT_URI,
-                                                       code)
-            user = yield self.oauth2_request(
-                self.test.get_url("/google/oauth2/userinfo"),
-                access_token=access["access_token"])
-            # return the user and access token as json
-            user["access_token"] = access["access_token"]
-            self.write(user)
-        else:
-            yield self.authorize_redirect(
-                redirect_uri=self._OAUTH_REDIRECT_URI,
-                client_id=self.settings['google_oauth']['key'],
-                client_secret=self.settings['google_oauth']['secret'],
-                scope=['profile', 'email'],
-                response_type='code',
-                extra_params={'prompt': 'select_account'})
-
-
-class GoogleOAuth2AuthorizeHandler(RequestHandler):
-    def get(self):
-        # issue a fake auth code and redirect to redirect_uri
-        code = 'fake-authorization-code'
-        self.redirect(url_concat(self.get_argument('redirect_uri'),
-                                 dict(code=code)))
-
-
-class GoogleOAuth2TokenHandler(RequestHandler):
-    def post(self):
-        assert self.get_argument('code') == 'fake-authorization-code'
-        # issue a fake token
-        self.finish({
-            'access_token': 'fake-access-token',
-            'expires_in': 'never-expires'
-        })
-
-
-class GoogleOAuth2UserinfoHandler(RequestHandler):
-    def get(self):
-        assert self.get_argument('access_token') == 'fake-access-token'
-        # return a fake user
-        self.finish({
-            'name': 'Foo',
-            'email': 'foo@example.com'
-        })
-
-
-class GoogleOAuth2Test(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application(
-            [
-                # test endpoints
-                ('/client/login', GoogleLoginHandler, dict(test=self)),
-
-                # simulated google authorization server endpoints
-                ('/google/oauth2/authorize', GoogleOAuth2AuthorizeHandler),
-                ('/google/oauth2/token', GoogleOAuth2TokenHandler),
-                ('/google/oauth2/userinfo', GoogleOAuth2UserinfoHandler),
-            ],
-            google_oauth={
-                "key": 'fake_google_client_id',
-                "secret": 'fake_google_client_secret'
-            })
-
-    def test_google_login(self):
-        response = self.fetch('/client/login')
-        self.assertDictEqual({
-            u'name': u'Foo',
-            u'email': u'foo@example.com',
-            u'access_token': u'fake-access-token',
-        }, json_decode(response.body))
diff --git a/salt/ext/tornado/test/concurrent_test.py b/salt/ext/tornado/test/concurrent_test.py
deleted file mode 100644
index 0f60ae5ea39..00000000000
--- a/salt/ext/tornado/test/concurrent_test.py
+++ /dev/null
@@ -1,436 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2012 Facebook
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-# pylint: skip-file
-from __future__ import absolute_import, division, print_function
-
-import gc
-import logging
-import re
-import socket
-import sys
-import traceback
-
-from salt.ext.tornado.concurrent import Future, return_future, ReturnValueIgnoredError, run_on_executor
-from salt.ext.tornado.escape import utf8, to_unicode
-from salt.ext.tornado import gen
-from salt.ext.tornado.iostream import IOStream
-from salt.ext.tornado.log import app_log
-from salt.ext.tornado import stack_context
-from salt.ext.tornado.tcpserver import TCPServer
-from salt.ext.tornado.testing import AsyncTestCase, ExpectLog, LogTrapTestCase, bind_unused_port, gen_test
-from salt.ext.tornado.test.util import unittest
-
-
-try:
-    from concurrent import futures
-except ImportError:
-    futures = None
-
-
-class ReturnFutureTest(AsyncTestCase):
-    @return_future
-    def sync_future(self, callback):
-        callback(42)
-
-    @return_future
-    def async_future(self, callback):
-        self.io_loop.add_callback(callback, 42)
-
-    @return_future
-    def immediate_failure(self, callback):
-        1 / 0
-
-    @return_future
-    def delayed_failure(self, callback):
-        self.io_loop.add_callback(lambda: 1 / 0)
-
-    @return_future
-    def return_value(self, callback):
-        # Note that the result of both running the callback and returning
-        # a value (or raising an exception) is unspecified; with current
-        # implementations the last event prior to callback resolution wins.
-        return 42
-
-    @return_future
-    def no_result_future(self, callback):
-        callback()
-
-    def test_immediate_failure(self):
-        with self.assertRaises(ZeroDivisionError):
-            # The caller sees the error just like a normal function.
-            self.immediate_failure(callback=self.stop)
-        # The callback is not run because the function failed synchronously.
-        self.io_loop.add_timeout(self.io_loop.time() + 0.05, self.stop)
-        result = self.wait()
-        self.assertIs(result, None)
-
-    def test_return_value(self):
-        with self.assertRaises(ReturnValueIgnoredError):
-            self.return_value(callback=self.stop)
-
-    def test_callback_kw(self):
-        future = self.sync_future(callback=self.stop)
-        result = self.wait()
-        self.assertEqual(result, 42)
-        self.assertEqual(future.result(), 42)
-
-    def test_callback_positional(self):
-        # When the callback is passed in positionally, future_wrap shouldn't
-        # add another callback in the kwargs.
-        future = self.sync_future(self.stop)
-        result = self.wait()
-        self.assertEqual(result, 42)
-        self.assertEqual(future.result(), 42)
-
-    def test_no_callback(self):
-        future = self.sync_future()
-        self.assertEqual(future.result(), 42)
-
-    def test_none_callback_kw(self):
-        # explicitly pass None as callback
-        future = self.sync_future(callback=None)
-        self.assertEqual(future.result(), 42)
-
-    def test_none_callback_pos(self):
-        future = self.sync_future(None)
-        self.assertEqual(future.result(), 42)
-
-    def test_async_future(self):
-        future = self.async_future()
-        self.assertFalse(future.done())
-        self.io_loop.add_future(future, self.stop)
-        future2 = self.wait()
-        self.assertIs(future, future2)
-        self.assertEqual(future.result(), 42)
-
-    @gen_test
-    def test_async_future_gen(self):
-        result = yield self.async_future()
-        self.assertEqual(result, 42)
-
-    def test_delayed_failure(self):
-        future = self.delayed_failure()
-        self.io_loop.add_future(future, self.stop)
-        future2 = self.wait()
-        self.assertIs(future, future2)
-        with self.assertRaises(ZeroDivisionError):
-            future.result()
-
-    def test_kw_only_callback(self):
-        @return_future
-        def f(**kwargs):
-            kwargs['callback'](42)
-        future = f()
-        self.assertEqual(future.result(), 42)
-
-    def test_error_in_callback(self):
-        self.sync_future(callback=lambda future: 1 / 0)
-        # The exception gets caught by our StackContext and will be re-raised
-        # when we wait.
-        self.assertRaises(ZeroDivisionError, self.wait)
-
-    def test_no_result_future(self):
-        future = self.no_result_future(self.stop)
-        result = self.wait()
-        self.assertIs(result, None)
-        # result of this future is undefined, but not an error
-        future.result()
-
-    def test_no_result_future_callback(self):
-        future = self.no_result_future(callback=lambda: self.stop())
-        result = self.wait()
-        self.assertIs(result, None)
-        future.result()
-
-    @gen_test
-    def test_future_traceback(self):
-        @return_future
-        @gen.engine
-        def f(callback):
-            yield gen.Task(self.io_loop.add_callback)
-            try:
-                1 / 0
-            except ZeroDivisionError:
-                self.expected_frame = traceback.extract_tb(
-                    sys.exc_info()[2], limit=1)[0]
-                raise
-        try:
-            yield f()
-            self.fail("didn't get expected exception")
-        except ZeroDivisionError:
-            tb = traceback.extract_tb(sys.exc_info()[2])
-            self.assertIn(self.expected_frame, tb)
-
-    @gen_test
-    def test_uncaught_exception_log(self):
-        @gen.coroutine
-        def f():
-            yield gen.moment
-            1 / 0
-
-        g = f()
-
-        with ExpectLog(app_log,
-                       "(?s)Future.* exception was never retrieved:"
-                       ".*ZeroDivisionError"):
-            yield gen.moment
-            yield gen.moment
-            del g
-            gc.collect()  # for PyPy
-
-
-# The following series of classes demonstrate and test various styles
-# of use, with and without generators and futures.
-
-
-class CapServer(TCPServer):
-    def handle_stream(self, stream, address):
-        logging.info("handle_stream")
-        self.stream = stream
-        self.stream.read_until(b"\n", self.handle_read)
-
-    def handle_read(self, data):
-        logging.info("handle_read")
-        data = to_unicode(data)
-        if data == data.upper():
-            self.stream.write(b"error\talready capitalized\n")
-        else:
-            # data already has \n
-            self.stream.write(utf8("ok\t%s" % data.upper()))
-        self.stream.close()
-
-
-class CapError(Exception):
-    pass
-
-
-class BaseCapClient(object):
-    def __init__(self, port, io_loop):
-        self.port = port
-        self.io_loop = io_loop
-
-    def process_response(self, data):
-        status, message = re.match('(.*)\t(.*)\n', to_unicode(data)).groups()
-        if status == 'ok':
-            return message
-        else:
-            raise CapError(message)
-
-
-class ManualCapClient(BaseCapClient):
-    def capitalize(self, request_data, callback=None):
-        logging.info("capitalize")
-        self.request_data = request_data
-        self.stream = IOStream(socket.socket(), io_loop=self.io_loop)
-        self.stream.connect(('127.0.0.1', self.port),
-                            callback=self.handle_connect)
-        self.future = Future()
-        if callback is not None:
-            self.future.add_done_callback(
-                stack_context.wrap(lambda future: callback(future.result())))
-        return self.future
-
-    def handle_connect(self):
-        logging.info("handle_connect")
-        self.stream.write(utf8(self.request_data + "\n"))
-        self.stream.read_until(b'\n', callback=self.handle_read)
-
-    def handle_read(self, data):
-        logging.info("handle_read")
-        self.stream.close()
-        try:
-            self.future.set_result(self.process_response(data))
-        except CapError as e:
-            self.future.set_exception(e)
-
-
-class DecoratorCapClient(BaseCapClient):
-    @return_future
-    def capitalize(self, request_data, callback):
-        logging.info("capitalize")
-        self.request_data = request_data
-        self.stream = IOStream(socket.socket(), io_loop=self.io_loop)
-        self.stream.connect(('127.0.0.1', self.port),
-                            callback=self.handle_connect)
-        self.callback = callback
-
-    def handle_connect(self):
-        logging.info("handle_connect")
-        self.stream.write(utf8(self.request_data + "\n"))
-        self.stream.read_until(b'\n', callback=self.handle_read)
-
-    def handle_read(self, data):
-        logging.info("handle_read")
-        self.stream.close()
-        self.callback(self.process_response(data))
-
-
-class GeneratorCapClient(BaseCapClient):
-    @return_future
-    @gen.engine
-    def capitalize(self, request_data, callback):
-        logging.info('capitalize')
-        stream = IOStream(socket.socket(), io_loop=self.io_loop)
-        logging.info('connecting')
-        yield gen.Task(stream.connect, ('127.0.0.1', self.port))
-        stream.write(utf8(request_data + '\n'))
-        logging.info('reading')
-        data = yield gen.Task(stream.read_until, b'\n')
-        logging.info('returning')
-        stream.close()
-        callback(self.process_response(data))
-
-
-class ClientTestMixin(object):
-    def setUp(self):
-        super(ClientTestMixin, self).setUp()  # type: ignore
-        self.server = CapServer(io_loop=self.io_loop)
-        sock, port = bind_unused_port()
-        self.server.add_sockets([sock])
-        self.client = self.client_class(io_loop=self.io_loop, port=port)
-
-    def tearDown(self):
-        self.server.stop()
-        super(ClientTestMixin, self).tearDown()  # type: ignore
-
-    def test_callback(self):
-        self.client.capitalize("hello", callback=self.stop)
-        result = self.wait()
-        self.assertEqual(result, "HELLO")
-
-    def test_callback_error(self):
-        self.client.capitalize("HELLO", callback=self.stop)
-        self.assertRaisesRegexp(CapError, "already capitalized", self.wait)
-
-    def test_future(self):
-        future = self.client.capitalize("hello")
-        self.io_loop.add_future(future, self.stop)
-        self.wait()
-        self.assertEqual(future.result(), "HELLO")
-
-    def test_future_error(self):
-        future = self.client.capitalize("HELLO")
-        self.io_loop.add_future(future, self.stop)
-        self.wait()
-        self.assertRaisesRegexp(CapError, "already capitalized", future.result)
-
-    def test_generator(self):
-        @gen.engine
-        def f():
-            result = yield self.client.capitalize("hello")
-            self.assertEqual(result, "HELLO")
-            self.stop()
-        f()
-        self.wait()
-
-    def test_generator_error(self):
-        @gen.engine
-        def f():
-            with self.assertRaisesRegexp(CapError, "already capitalized"):
-                yield self.client.capitalize("HELLO")
-            self.stop()
-        f()
-        self.wait()
-
-
-class ManualClientTest(ClientTestMixin, AsyncTestCase, LogTrapTestCase):
-    client_class = ManualCapClient
-
-
-class DecoratorClientTest(ClientTestMixin, AsyncTestCase, LogTrapTestCase):
-    client_class = DecoratorCapClient
-
-
-class GeneratorClientTest(ClientTestMixin, AsyncTestCase, LogTrapTestCase):
-    client_class = GeneratorCapClient
-
-
-@unittest.skipIf(futures is None, "concurrent.futures module not present")
-class RunOnExecutorTest(AsyncTestCase):
-    @gen_test
-    def test_no_calling(self):
-        class Object(object):
-            def __init__(self, io_loop):
-                self.io_loop = io_loop
-                self.executor = futures.thread.ThreadPoolExecutor(1)
-
-            @run_on_executor
-            def f(self):
-                return 42
-
-        o = Object(io_loop=self.io_loop)
-        answer = yield o.f()
-        self.assertEqual(answer, 42)
-
-    @gen_test
-    def test_call_with_no_args(self):
-        class Object(object):
-            def __init__(self, io_loop):
-                self.io_loop = io_loop
-                self.executor = futures.thread.ThreadPoolExecutor(1)
-
-            @run_on_executor()
-            def f(self):
-                return 42
-
-        o = Object(io_loop=self.io_loop)
-        answer = yield o.f()
-        self.assertEqual(answer, 42)
-
-    @gen_test
-    def test_call_with_io_loop(self):
-        class Object(object):
-            def __init__(self, io_loop):
-                self._io_loop = io_loop
-                self.executor = futures.thread.ThreadPoolExecutor(1)
-
-            @run_on_executor(io_loop='_io_loop')
-            def f(self):
-                return 42
-
-        o = Object(io_loop=self.io_loop)
-        answer = yield o.f()
-        self.assertEqual(answer, 42)
-
-    @gen_test
-    def test_call_with_executor(self):
-        class Object(object):
-            def __init__(self, io_loop):
-                self.io_loop = io_loop
-                self.__executor = futures.thread.ThreadPoolExecutor(1)
-
-            @run_on_executor(executor='_Object__executor')
-            def f(self):
-                return 42
-
-        o = Object(io_loop=self.io_loop)
-        answer = yield o.f()
-        self.assertEqual(answer, 42)
-
-    @gen_test
-    def test_call_with_both(self):
-        class Object(object):
-            def __init__(self, io_loop):
-                self._io_loop = io_loop
-                self.__executor = futures.thread.ThreadPoolExecutor(1)
-
-            @run_on_executor(io_loop='_io_loop', executor='_Object__executor')
-            def f(self):
-                return 42
-
-        o = Object(io_loop=self.io_loop)
-        answer = yield o.f()
-        self.assertEqual(answer, 42)
diff --git a/salt/ext/tornado/test/csv_translations/fr_FR.csv b/salt/ext/tornado/test/csv_translations/fr_FR.csv
deleted file mode 100644
index 6321b6e7c08..00000000000
--- a/salt/ext/tornado/test/csv_translations/fr_FR.csv
+++ /dev/null
@@ -1 +0,0 @@
-"school","école"
diff --git a/salt/ext/tornado/test/curl_httpclient_test.py b/salt/ext/tornado/test/curl_httpclient_test.py
deleted file mode 100644
index ba6cc06e7c8..00000000000
--- a/salt/ext/tornado/test/curl_httpclient_test.py
+++ /dev/null
@@ -1,135 +0,0 @@
-# coding: utf-8
-# pylint: skip-file
-from __future__ import absolute_import, division, print_function
-
-from hashlib import md5
-
-from salt.ext.tornado.escape import utf8
-from salt.ext.tornado.httpclient import HTTPRequest
-from salt.ext.tornado.stack_context import ExceptionStackContext
-from salt.ext.tornado.testing import AsyncHTTPTestCase
-from salt.ext.tornado.test import httpclient_test
-from salt.ext.tornado.test.util import unittest
-from salt.ext.tornado.web import Application, RequestHandler
-
-
-try:
-    import pycurl  # type: ignore
-except ImportError:
-    pycurl = None
-
-if pycurl is not None:
-    from salt.ext.tornado.curl_httpclient import CurlAsyncHTTPClient
-
-
-@unittest.skipIf(pycurl is None, "pycurl module not present")
-class CurlHTTPClientCommonTestCase(httpclient_test.HTTPClientCommonTestCase):
-    def get_http_client(self):
-        client = CurlAsyncHTTPClient(io_loop=self.io_loop,
-                                     defaults=dict(allow_ipv6=False))
-        # make sure AsyncHTTPClient magic doesn't give us the wrong class
-        self.assertTrue(isinstance(client, CurlAsyncHTTPClient))
-        return client
-
-
-class DigestAuthHandler(RequestHandler):
-    def get(self):
-        realm = 'test'
-        opaque = 'asdf'
-        # Real implementations would use a random nonce.
-        nonce = "1234"
-        username = 'foo'
-        password = 'bar'
-
-        auth_header = self.request.headers.get('Authorization', None)
-        if auth_header is not None:
-            auth_mode, params = auth_header.split(' ', 1)
-            assert auth_mode == 'Digest'
-            param_dict = {}
-            for pair in params.split(','):
-                k, v = pair.strip().split('=', 1)
-                if v[0] == '"' and v[-1] == '"':
-                    v = v[1:-1]
-                param_dict[k] = v
-            assert param_dict['realm'] == realm
-            assert param_dict['opaque'] == opaque
-            assert param_dict['nonce'] == nonce
-            assert param_dict['username'] == username
-            assert param_dict['uri'] == self.request.path
-            h1 = md5(utf8('%s:%s:%s' % (username, realm, password))).hexdigest()
-            h2 = md5(utf8('%s:%s' % (self.request.method,
-                                     self.request.path))).hexdigest()
-            digest = md5(utf8('%s:%s:%s' % (h1, nonce, h2))).hexdigest()
-            if digest == param_dict['response']:
-                self.write('ok')
-            else:
-                self.write('fail')
-        else:
-            self.set_status(401)
-            self.set_header('WWW-Authenticate',
-                            'Digest realm="%s", nonce="%s", opaque="%s"' %
-                            (realm, nonce, opaque))
-
-
-class CustomReasonHandler(RequestHandler):
-    def get(self):
-        self.set_status(200, "Custom reason")
-
-
-class CustomFailReasonHandler(RequestHandler):
-    def get(self):
-        self.set_status(400, "Custom reason")
-
-
-@unittest.skipIf(pycurl is None, "pycurl module not present")
-class CurlHTTPClientTestCase(AsyncHTTPTestCase):
-    def setUp(self):
-        super(CurlHTTPClientTestCase, self).setUp()
-        self.http_client = self.create_client()
-
-    def get_app(self):
-        return Application([
-            ('/digest', DigestAuthHandler),
-            ('/custom_reason', CustomReasonHandler),
-            ('/custom_fail_reason', CustomFailReasonHandler),
-        ])
-
-    def create_client(self, **kwargs):
-        return CurlAsyncHTTPClient(self.io_loop, force_instance=True,
-                                   defaults=dict(allow_ipv6=False),
-                                   **kwargs)
-
-    def test_prepare_curl_callback_stack_context(self):
-        exc_info = []
-
-        def error_handler(typ, value, tb):
-            exc_info.append((typ, value, tb))
-            self.stop()
-            return True
-
-        with ExceptionStackContext(error_handler):
-            request = HTTPRequest(self.get_url('/'),
-                                  prepare_curl_callback=lambda curl: 1 / 0)
-        self.http_client.fetch(request, callback=self.stop)
-        self.wait()
-        self.assertEqual(1, len(exc_info))
-        self.assertIs(exc_info[0][0], ZeroDivisionError)
-
-    def test_digest_auth(self):
-        response = self.fetch('/digest', auth_mode='digest',
-                              auth_username='foo', auth_password='bar')
-        self.assertEqual(response.body, b'ok')
-
-    def test_custom_reason(self):
-        response = self.fetch('/custom_reason')
-        self.assertEqual(response.reason, "Custom reason")
-
-    def test_fail_custom_reason(self):
-        response = self.fetch('/custom_fail_reason')
-        self.assertEqual(str(response.error), "HTTP 400: Custom reason")
-
-    def test_failed_setup(self):
-        self.http_client = self.create_client(max_clients=1)
-        for i in range(5):
-            response = self.fetch(u'/ユニコード')
-            self.assertIsNot(response.error, None)
diff --git a/salt/ext/tornado/test/escape_test.py b/salt/ext/tornado/test/escape_test.py
deleted file mode 100644
index 27b4784ae05..00000000000
--- a/salt/ext/tornado/test/escape_test.py
+++ /dev/null
@@ -1,246 +0,0 @@
-#!/usr/bin/env python
-# pylint: skip-file
-
-
-from __future__ import absolute_import, division, print_function
-import salt.ext.tornado.escape
-
-from salt.ext.tornado.escape import utf8, xhtml_escape, xhtml_unescape, url_escape, url_unescape, to_unicode, json_decode, json_encode, squeeze, recursive_unicode
-from salt.ext.tornado.util import unicode_type
-from salt.ext.tornado.test.util import unittest
-
-linkify_tests = [
-    # (input, linkify_kwargs, expected_output)
-
-    ("hello http://world.com/!", {},
-     u'hello http://world.com/!'),
-
-    ("hello http://world.com/with?param=true&stuff=yes", {},
-     u'hello http://world.com/with?param=true&stuff=yes'),
-
-    # an opened paren followed by many chars killed Gruber's regex
-    ("http://url.com/w(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {},
-     u'http://url.com/w(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'),
-
-    # as did too many dots at the end
-    ("http://url.com/withmany.......................................", {},
-     u'http://url.com/withmany.......................................'),
-
-    ("http://url.com/withmany((((((((((((((((((((((((((((((((((a)", {},
-     u'http://url.com/withmany((((((((((((((((((((((((((((((((((a)'),
-
-    # some examples from http://daringfireball.net/2009/11/liberal_regex_for_matching_urls
-    # plus a fex extras (such as multiple parentheses).
-    ("http://foo.com/blah_blah", {},
-     u'http://foo.com/blah_blah'),
-
-    ("http://foo.com/blah_blah/", {},
-     u'http://foo.com/blah_blah/'),
-
-    ("(Something like http://foo.com/blah_blah)", {},
-     u'(Something like http://foo.com/blah_blah)'),
-
-    ("http://foo.com/blah_blah_(wikipedia)", {},
-     u'http://foo.com/blah_blah_(wikipedia)'),
-
-    ("http://foo.com/blah_(blah)_(wikipedia)_blah", {},
-     u'http://foo.com/blah_(blah)_(wikipedia)_blah'),
-
-    ("(Something like http://foo.com/blah_blah_(wikipedia))", {},
-     u'(Something like http://foo.com/blah_blah_(wikipedia))'),
-
-    ("http://foo.com/blah_blah.", {},
-     u'http://foo.com/blah_blah.'),
-
-    ("http://foo.com/blah_blah/.", {},
-     u'http://foo.com/blah_blah/.'),
-
-    ("", {},
-     u'<http://foo.com/blah_blah>'),
-
-    ("", {},
-     u'<http://foo.com/blah_blah/>'),
-
-    ("http://foo.com/blah_blah,", {},
-     u'http://foo.com/blah_blah,'),
-
-    ("http://www.example.com/wpstyle/?p=364.", {},
-     u'http://www.example.com/wpstyle/?p=364.'),
-
-    ("rdar://1234",
-     {"permitted_protocols": ["http", "rdar"]},
-     u'rdar://1234'),
-
-    ("rdar:/1234",
-     {"permitted_protocols": ["rdar"]},
-     u'rdar:/1234'),
-
-    ("http://userid:password@example.com:8080", {},
-     u'http://userid:password@example.com:8080'),
-
-    ("http://userid@example.com", {},
-     u'http://userid@example.com'),
-
-    ("http://userid@example.com:8080", {},
-     u'http://userid@example.com:8080'),
-
-    ("http://userid:password@example.com", {},
-     u'http://userid:password@example.com'),
-
-    ("message://%3c330e7f8409726r6a4ba78dkf1fd71420c1bf6ff@mail.gmail.com%3e",
-     {"permitted_protocols": ["http", "message"]},
-     u'message://%3c330e7f8409726r6a4ba78dkf1fd71420c1bf6ff@mail.gmail.com%3e'),
-
-    (u"http://\u27a1.ws/\u4a39", {},
-     u'http://\u27a1.ws/\u4a39'),
-
-    ("http://example.com", {},
-     u'<tag>http://example.com</tag>'),
-
-    ("Just a www.example.com link.", {},
-     u'Just a www.example.com link.'),
-
-    ("Just a www.example.com link.",
-     {"require_protocol": True},
-     u'Just a www.example.com link.'),
-
-    ("A http://reallylong.com/link/that/exceedsthelenglimit.html",
-     {"require_protocol": True, "shorten": True},
-     u'A http://reallylong.com/link...'),
-
-    ("A http://reallylongdomainnamethatwillbetoolong.com/hi!",
-     {"shorten": True},
-     u'A http://reallylongdomainnametha...!'),
-
-    ("A file:///passwords.txt and http://web.com link", {},
-     u'A file:///passwords.txt and http://web.com link'),
-
-    ("A file:///passwords.txt and http://web.com link",
-     {"permitted_protocols": ["file"]},
-     u'A file:///passwords.txt and http://web.com link'),
-
-    ("www.external-link.com",
-     {"extra_params": 'rel="nofollow" class="external"'},
-     u'www.external-link.com'),
-
-    ("www.external-link.com and www.internal-link.com/blogs extra",
-     {"extra_params": lambda href: 'class="internal"' if href.startswith("http://www.internal-link.com") else 'rel="nofollow" class="external"'},
-     u'www.external-link.com and www.internal-link.com/blogs extra'),
-
-    ("www.external-link.com",
-     {"extra_params": lambda href: '    rel="nofollow" class="external"  '},
-     u'www.external-link.com'),
-]
-
-
-class EscapeTestCase(unittest.TestCase):
-    def test_linkify(self):
-        for text, kwargs, html in linkify_tests:
-            linked = salt.ext.tornado.escape.linkify(text, **kwargs)
-            self.assertEqual(linked, html)
-
-    def test_xhtml_escape(self):
-        tests = [
-            ("", "<foo>"),
-            (u"", u"<foo>"),
-            (b"", b"<foo>"),
-
-            ("<>&\"'", "<>&"'"),
-            ("&", "&amp;"),
-
-            (u"<\u00e9>", u"<\u00e9>"),
-            (b"<\xc3\xa9>", b"<\xc3\xa9>"),
-        ]
-        for unescaped, escaped in tests:
-            self.assertEqual(utf8(xhtml_escape(unescaped)), utf8(escaped))
-            self.assertEqual(utf8(unescaped), utf8(xhtml_unescape(escaped)))
-
-    def test_xhtml_unescape_numeric(self):
-        tests = [
-            ('foo bar', 'foo bar'),
-            ('foo bar', 'foo bar'),
-            ('foo bar', 'foo bar'),
-            ('foo઼bar', u'foo\u0abcbar'),
-            ('foo&#xyz;bar', 'foo&#xyz;bar'),  # invalid encoding
-            ('foo&#;bar', 'foo&#;bar'),        # invalid encoding
-            ('foo&#x;bar', 'foo&#x;bar'),      # invalid encoding
-        ]
-        for escaped, unescaped in tests:
-            self.assertEqual(unescaped, xhtml_unescape(escaped))
-
-    def test_url_escape_unicode(self):
-        tests = [
-            # byte strings are passed through as-is
-            (u'\u00e9'.encode('utf8'), '%C3%A9'),
-            (u'\u00e9'.encode('latin1'), '%E9'),
-
-            # unicode strings become utf8
-            (u'\u00e9', '%C3%A9'),
-        ]
-        for unescaped, escaped in tests:
-            self.assertEqual(url_escape(unescaped), escaped)
-
-    def test_url_unescape_unicode(self):
-        tests = [
-            ('%C3%A9', u'\u00e9', 'utf8'),
-            ('%C3%A9', u'\u00c3\u00a9', 'latin1'),
-            ('%C3%A9', utf8(u'\u00e9'), None),
-        ]
-        for escaped, unescaped, encoding in tests:
-            # input strings to url_unescape should only contain ascii
-            # characters, but make sure the function accepts both byte
-            # and unicode strings.
-            self.assertEqual(url_unescape(to_unicode(escaped), encoding), unescaped)
-            self.assertEqual(url_unescape(utf8(escaped), encoding), unescaped)
-
-    def test_url_escape_quote_plus(self):
-        unescaped = '+ #%'
-        plus_escaped = '%2B+%23%25'
-        escaped = '%2B%20%23%25'
-        self.assertEqual(url_escape(unescaped), plus_escaped)
-        self.assertEqual(url_escape(unescaped, plus=False), escaped)
-        self.assertEqual(url_unescape(plus_escaped), unescaped)
-        self.assertEqual(url_unescape(escaped, plus=False), unescaped)
-        self.assertEqual(url_unescape(plus_escaped, encoding=None),
-                         utf8(unescaped))
-        self.assertEqual(url_unescape(escaped, encoding=None, plus=False),
-                         utf8(unescaped))
-
-    def test_escape_return_types(self):
-        # On python2 the escape methods should generally return the same
-        # type as their argument
-        self.assertEqual(type(xhtml_escape("foo")), str)
-        self.assertEqual(type(xhtml_escape(u"foo")), unicode_type)
-
-    def test_json_decode(self):
-        # json_decode accepts both bytes and unicode, but strings it returns
-        # are always unicode.
-        self.assertEqual(json_decode(b'"foo"'), u"foo")
-        self.assertEqual(json_decode(u'"foo"'), u"foo")
-
-        # Non-ascii bytes are interpreted as utf8
-        self.assertEqual(json_decode(utf8(u'"\u00e9"')), u"\u00e9")
-
-    def test_json_encode(self):
-        # json deals with strings, not bytes.  On python 2 byte strings will
-        # convert automatically if they are utf8; on python 3 byte strings
-        # are not allowed.
-        self.assertEqual(json_decode(json_encode(u"\u00e9")), u"\u00e9")
-        if bytes is str:
-            self.assertEqual(json_decode(json_encode(utf8(u"\u00e9"))), u"\u00e9")
-            self.assertRaises(UnicodeDecodeError, json_encode, b"\xe9")
-
-    def test_squeeze(self):
-        self.assertEqual(squeeze(u'sequences     of    whitespace   chars'), u'sequences of whitespace chars')
-
-    def test_recursive_unicode(self):
-        tests = {
-            'dict': {b"foo": b"bar"},
-            'list': [b"foo", b"bar"],
-            'tuple': (b"foo", b"bar"),
-            'bytes': b"foo"
-        }
-        self.assertEqual(recursive_unicode(tests['dict']), {u"foo": u"bar"})
-        self.assertEqual(recursive_unicode(tests['list']), [u"foo", u"bar"])
-        self.assertEqual(recursive_unicode(tests['tuple']), (u"foo", u"bar"))
-        self.assertEqual(recursive_unicode(tests['bytes']), u"foo")
diff --git a/salt/ext/tornado/test/gen_test.py b/salt/ext/tornado/test/gen_test.py
deleted file mode 100644
index d5c8a629a29..00000000000
--- a/salt/ext/tornado/test/gen_test.py
+++ /dev/null
@@ -1,1468 +0,0 @@
-# pylint: skip-file
-from __future__ import absolute_import, division, print_function
-
-import gc
-import contextlib
-import datetime
-import functools
-import sys
-import textwrap
-import time
-import weakref
-
-from salt.ext.tornado.concurrent import return_future, Future
-from salt.ext.tornado.escape import url_escape
-from salt.ext.tornado.httpclient import AsyncHTTPClient
-from salt.ext.tornado.ioloop import IOLoop
-from salt.ext.tornado.log import app_log
-from salt.ext.tornado import stack_context
-from salt.ext.tornado.testing import AsyncHTTPTestCase, AsyncTestCase, ExpectLog, gen_test
-from salt.ext.tornado.test.util import unittest, skipOnTravis, skipBefore33, skipBefore35, skipNotCPython, exec_test
-from salt.ext.tornado.web import Application, RequestHandler, asynchronous, HTTPError
-
-from salt.ext.tornado import gen
-
-try:
-    from concurrent import futures
-except ImportError:
-    futures = None
-
-
-class GenEngineTest(AsyncTestCase):
-    def setUp(self):
-        super(GenEngineTest, self).setUp()
-        self.named_contexts = []
-
-    def named_context(self, name):
-        @contextlib.contextmanager
-        def context():
-            self.named_contexts.append(name)
-            try:
-                yield
-            finally:
-                self.assertEqual(self.named_contexts.pop(), name)
-        return context
-
-    def run_gen(self, f):
-        f()
-        return self.wait()
-
-    def delay_callback(self, iterations, callback, arg):
-        """Runs callback(arg) after a number of IOLoop iterations."""
-        if iterations == 0:
-            callback(arg)
-        else:
-            self.io_loop.add_callback(functools.partial(
-                self.delay_callback, iterations - 1, callback, arg))
-
-    @return_future
-    def async_future(self, result, callback):
-        self.io_loop.add_callback(callback, result)
-
-    @gen.coroutine
-    def async_exception(self, e):
-        yield gen.moment
-        raise e
-
-    def test_no_yield(self):
-        @gen.engine
-        def f():
-            self.stop()
-        self.run_gen(f)
-
-    def test_inline_cb(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback("k1"))()
-            res = yield gen.Wait("k1")
-            self.assertTrue(res is None)
-            self.stop()
-        self.run_gen(f)
-
-    def test_ioloop_cb(self):
-        @gen.engine
-        def f():
-            self.io_loop.add_callback((yield gen.Callback("k1")))
-            yield gen.Wait("k1")
-            self.stop()
-        self.run_gen(f)
-
-    def test_exception_phase1(self):
-        @gen.engine
-        def f():
-            1 / 0
-        self.assertRaises(ZeroDivisionError, self.run_gen, f)
-
-    def test_exception_phase2(self):
-        @gen.engine
-        def f():
-            self.io_loop.add_callback((yield gen.Callback("k1")))
-            yield gen.Wait("k1")
-            1 / 0
-        self.assertRaises(ZeroDivisionError, self.run_gen, f)
-
-    def test_exception_in_task_phase1(self):
-        def fail_task(callback):
-            1 / 0
-
-        @gen.engine
-        def f():
-            try:
-                yield gen.Task(fail_task)
-                raise Exception("did not get expected exception")
-            except ZeroDivisionError:
-                self.stop()
-        self.run_gen(f)
-
-    def test_exception_in_task_phase2(self):
-        # This is the case that requires the use of stack_context in gen.engine
-        def fail_task(callback):
-            self.io_loop.add_callback(lambda: 1 / 0)
-
-        @gen.engine
-        def f():
-            try:
-                yield gen.Task(fail_task)
-                raise Exception("did not get expected exception")
-            except ZeroDivisionError:
-                self.stop()
-        self.run_gen(f)
-
-    def test_with_arg(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback("k1"))(42)
-            res = yield gen.Wait("k1")
-            self.assertEqual(42, res)
-            self.stop()
-        self.run_gen(f)
-
-    def test_with_arg_tuple(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback((1, 2)))((3, 4))
-            res = yield gen.Wait((1, 2))
-            self.assertEqual((3, 4), res)
-            self.stop()
-        self.run_gen(f)
-
-    def test_key_reuse(self):
-        @gen.engine
-        def f():
-            yield gen.Callback("k1")
-            yield gen.Callback("k1")
-            self.stop()
-        self.assertRaises(gen.KeyReuseError, self.run_gen, f)
-
-    def test_key_reuse_tuple(self):
-        @gen.engine
-        def f():
-            yield gen.Callback((1, 2))
-            yield gen.Callback((1, 2))
-            self.stop()
-        self.assertRaises(gen.KeyReuseError, self.run_gen, f)
-
-    def test_key_mismatch(self):
-        @gen.engine
-        def f():
-            yield gen.Callback("k1")
-            yield gen.Wait("k2")
-            self.stop()
-        self.assertRaises(gen.UnknownKeyError, self.run_gen, f)
-
-    def test_key_mismatch_tuple(self):
-        @gen.engine
-        def f():
-            yield gen.Callback((1, 2))
-            yield gen.Wait((2, 3))
-            self.stop()
-        self.assertRaises(gen.UnknownKeyError, self.run_gen, f)
-
-    def test_leaked_callback(self):
-        @gen.engine
-        def f():
-            yield gen.Callback("k1")
-            self.stop()
-        self.assertRaises(gen.LeakedCallbackError, self.run_gen, f)
-
-    def test_leaked_callback_tuple(self):
-        @gen.engine
-        def f():
-            yield gen.Callback((1, 2))
-            self.stop()
-        self.assertRaises(gen.LeakedCallbackError, self.run_gen, f)
-
-    def test_parallel_callback(self):
-        @gen.engine
-        def f():
-            for k in range(3):
-                self.io_loop.add_callback((yield gen.Callback(k)))
-            yield gen.Wait(1)
-            self.io_loop.add_callback((yield gen.Callback(3)))
-            yield gen.Wait(0)
-            yield gen.Wait(3)
-            yield gen.Wait(2)
-            self.stop()
-        self.run_gen(f)
-
-    def test_bogus_yield(self):
-        @gen.engine
-        def f():
-            yield 42
-        self.assertRaises(gen.BadYieldError, self.run_gen, f)
-
-    def test_bogus_yield_tuple(self):
-        @gen.engine
-        def f():
-            yield (1, 2)
-        self.assertRaises(gen.BadYieldError, self.run_gen, f)
-
-    def test_reuse(self):
-        @gen.engine
-        def f():
-            self.io_loop.add_callback((yield gen.Callback(0)))
-            yield gen.Wait(0)
-            self.stop()
-        self.run_gen(f)
-        self.run_gen(f)
-
-    def test_task(self):
-        @gen.engine
-        def f():
-            yield gen.Task(self.io_loop.add_callback)
-            self.stop()
-        self.run_gen(f)
-
-    def test_wait_all(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback("k1"))("v1")
-            (yield gen.Callback("k2"))("v2")
-            results = yield gen.WaitAll(["k1", "k2"])
-            self.assertEqual(results, ["v1", "v2"])
-            self.stop()
-        self.run_gen(f)
-
-    def test_exception_in_yield(self):
-        @gen.engine
-        def f():
-            try:
-                yield gen.Wait("k1")
-                raise Exception("did not get expected exception")
-            except gen.UnknownKeyError:
-                pass
-            self.stop()
-        self.run_gen(f)
-
-    def test_resume_after_exception_in_yield(self):
-        @gen.engine
-        def f():
-            try:
-                yield gen.Wait("k1")
-                raise Exception("did not get expected exception")
-            except gen.UnknownKeyError:
-                pass
-            (yield gen.Callback("k2"))("v2")
-            self.assertEqual((yield gen.Wait("k2")), "v2")
-            self.stop()
-        self.run_gen(f)
-
-    def test_orphaned_callback(self):
-        @gen.engine
-        def f():
-            self.orphaned_callback = yield gen.Callback(1)
-        try:
-            self.run_gen(f)
-            raise Exception("did not get expected exception")
-        except gen.LeakedCallbackError:
-            pass
-        self.orphaned_callback()
-
-    def test_none(self):
-        @gen.engine
-        def f():
-            yield None
-            self.stop()
-        self.run_gen(f)
-
-    def test_multi(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback("k1"))("v1")
-            (yield gen.Callback("k2"))("v2")
-            results = yield [gen.Wait("k1"), gen.Wait("k2")]
-            self.assertEqual(results, ["v1", "v2"])
-            self.stop()
-        self.run_gen(f)
-
-    def test_multi_dict(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback("k1"))("v1")
-            (yield gen.Callback("k2"))("v2")
-            results = yield dict(foo=gen.Wait("k1"), bar=gen.Wait("k2"))
-            self.assertEqual(results, dict(foo="v1", bar="v2"))
-            self.stop()
-        self.run_gen(f)
-
-    # The following tests explicitly run with both gen.Multi
-    # and gen.multi_future (Task returns a Future, so it can be used
-    # with either).
-    def test_multi_yieldpoint_delayed(self):
-        @gen.engine
-        def f():
-            # callbacks run at different times
-            responses = yield gen.Multi([
-                gen.Task(self.delay_callback, 3, arg="v1"),
-                gen.Task(self.delay_callback, 1, arg="v2"),
-            ])
-            self.assertEqual(responses, ["v1", "v2"])
-            self.stop()
-        self.run_gen(f)
-
-    def test_multi_yieldpoint_dict_delayed(self):
-        @gen.engine
-        def f():
-            # callbacks run at different times
-            responses = yield gen.Multi(dict(
-                foo=gen.Task(self.delay_callback, 3, arg="v1"),
-                bar=gen.Task(self.delay_callback, 1, arg="v2"),
-            ))
-            self.assertEqual(responses, dict(foo="v1", bar="v2"))
-            self.stop()
-        self.run_gen(f)
-
-    def test_multi_future_delayed(self):
-        @gen.engine
-        def f():
-            # callbacks run at different times
-            responses = yield gen.multi_future([
-                gen.Task(self.delay_callback, 3, arg="v1"),
-                gen.Task(self.delay_callback, 1, arg="v2"),
-            ])
-            self.assertEqual(responses, ["v1", "v2"])
-            self.stop()
-        self.run_gen(f)
-
-    def test_multi_future_dict_delayed(self):
-        @gen.engine
-        def f():
-            # callbacks run at different times
-            responses = yield gen.multi_future(dict(
-                foo=gen.Task(self.delay_callback, 3, arg="v1"),
-                bar=gen.Task(self.delay_callback, 1, arg="v2"),
-            ))
-            self.assertEqual(responses, dict(foo="v1", bar="v2"))
-            self.stop()
-        self.run_gen(f)
-
-    @skipOnTravis
-    @gen_test
-    def test_multi_performance(self):
-        # Yielding a list used to have quadratic performance; make
-        # sure a large list stays reasonable.  On my laptop a list of
-        # 2000 used to take 1.8s, now it takes 0.12.
-        start = time.time()
-        yield [gen.Task(self.io_loop.add_callback) for i in range(2000)]
-        end = time.time()
-        self.assertLess(end - start, 1.0)
-
-    @gen_test
-    def test_multi_empty(self):
-        # Empty lists or dicts should return the same type.
-        x = yield []
-        self.assertTrue(isinstance(x, list))
-        y = yield {}
-        self.assertTrue(isinstance(y, dict))
-
-    @gen_test
-    def test_multi_mixed_types(self):
-        # A YieldPoint (Wait) and Future (Task) can be combined
-        # (and use the YieldPoint codepath)
-        (yield gen.Callback("k1"))("v1")
-        responses = yield [gen.Wait("k1"),
-                           gen.Task(self.delay_callback, 3, arg="v2")]
-        self.assertEqual(responses, ["v1", "v2"])
-
-    @gen_test
-    def test_future(self):
-        result = yield self.async_future(1)
-        self.assertEqual(result, 1)
-
-    @gen_test
-    def test_multi_future(self):
-        results = yield [self.async_future(1), self.async_future(2)]
-        self.assertEqual(results, [1, 2])
-
-    @gen_test
-    def test_multi_future_duplicate(self):
-        f = self.async_future(2)
-        results = yield [self.async_future(1), f, self.async_future(3), f]
-        self.assertEqual(results, [1, 2, 3, 2])
-
-    @gen_test
-    def test_multi_dict_future(self):
-        results = yield dict(foo=self.async_future(1), bar=self.async_future(2))
-        self.assertEqual(results, dict(foo=1, bar=2))
-
-    @gen_test
-    def test_multi_exceptions(self):
-        with ExpectLog(app_log, "Multiple exceptions in yield list"):
-            with self.assertRaises(RuntimeError) as cm:
-                yield gen.Multi([self.async_exception(RuntimeError("error 1")),
-                                 self.async_exception(RuntimeError("error 2"))])
-        self.assertEqual(str(cm.exception), "error 1")
-
-        # With only one exception, no error is logged.
-        with self.assertRaises(RuntimeError):
-            yield gen.Multi([self.async_exception(RuntimeError("error 1")),
-                             self.async_future(2)])
-
-        # Exception logging may be explicitly quieted.
-        with self.assertRaises(RuntimeError):
-            yield gen.Multi([self.async_exception(RuntimeError("error 1")),
-                             self.async_exception(RuntimeError("error 2"))],
-                            quiet_exceptions=RuntimeError)
-
-    @gen_test
-    def test_multi_future_exceptions(self):
-        with ExpectLog(app_log, "Multiple exceptions in yield list"):
-            with self.assertRaises(RuntimeError) as cm:
-                yield [self.async_exception(RuntimeError("error 1")),
-                       self.async_exception(RuntimeError("error 2"))]
-        self.assertEqual(str(cm.exception), "error 1")
-
-        # With only one exception, no error is logged.
-        with self.assertRaises(RuntimeError):
-            yield [self.async_exception(RuntimeError("error 1")),
-                   self.async_future(2)]
-
-        # Exception logging may be explicitly quieted.
-        with self.assertRaises(RuntimeError):
-            yield gen.multi_future(
-                [self.async_exception(RuntimeError("error 1")),
-                 self.async_exception(RuntimeError("error 2"))],
-                quiet_exceptions=RuntimeError)
-
-    def test_arguments(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback("noargs"))()
-            self.assertEqual((yield gen.Wait("noargs")), None)
-            (yield gen.Callback("1arg"))(42)
-            self.assertEqual((yield gen.Wait("1arg")), 42)
-
-            (yield gen.Callback("kwargs"))(value=42)
-            result = yield gen.Wait("kwargs")
-            self.assertTrue(isinstance(result, gen.Arguments))
-            self.assertEqual(((), dict(value=42)), result)
-            self.assertEqual(dict(value=42), result.kwargs)
-
-            (yield gen.Callback("2args"))(42, 43)
-            result = yield gen.Wait("2args")
-            self.assertTrue(isinstance(result, gen.Arguments))
-            self.assertEqual(((42, 43), {}), result)
-            self.assertEqual((42, 43), result.args)
-
-            def task_func(callback):
-                callback(None, error="foo")
-            result = yield gen.Task(task_func)
-            self.assertTrue(isinstance(result, gen.Arguments))
-            self.assertEqual(((None,), dict(error="foo")), result)
-
-            self.stop()
-        self.run_gen(f)
-
-    def test_stack_context_leak(self):
-        # regression test: repeated invocations of a gen-based
-        # function should not result in accumulated stack_contexts
-        def _stack_depth():
-            head = stack_context._state.contexts[1]
-            length = 0
-
-            while head is not None:
-                length += 1
-                head = head.old_contexts[1]
-
-            return length
-
-        @gen.engine
-        def inner(callback):
-            yield gen.Task(self.io_loop.add_callback)
-            callback()
-
-        @gen.engine
-        def outer():
-            for i in range(10):
-                yield gen.Task(inner)
-
-            stack_increase = _stack_depth() - initial_stack_depth
-            self.assertTrue(stack_increase <= 2)
-            self.stop()
-        initial_stack_depth = _stack_depth()
-        self.run_gen(outer)
-
-    def test_stack_context_leak_exception(self):
-        # same as previous, but with a function that exits with an exception
-        @gen.engine
-        def inner(callback):
-            yield gen.Task(self.io_loop.add_callback)
-            1 / 0
-
-        @gen.engine
-        def outer():
-            for i in range(10):
-                try:
-                    yield gen.Task(inner)
-                except ZeroDivisionError:
-                    pass
-            stack_increase = len(stack_context._state.contexts) - initial_stack_depth
-            self.assertTrue(stack_increase <= 2)
-            self.stop()
-        initial_stack_depth = len(stack_context._state.contexts)
-        self.run_gen(outer)
-
-    def function_with_stack_context(self, callback):
-        # Technically this function should stack_context.wrap its callback
-        # upon entry.  However, it is very common for this step to be
-        # omitted.
-        def step2():
-            self.assertEqual(self.named_contexts, ['a'])
-            self.io_loop.add_callback(callback)
-
-        with stack_context.StackContext(self.named_context('a')):
-            self.io_loop.add_callback(step2)
-
-    @gen_test
-    def test_wait_transfer_stack_context(self):
-        # Wait should not pick up contexts from where callback was invoked,
-        # even if that function improperly fails to wrap its callback.
-        cb = yield gen.Callback('k1')
-        self.function_with_stack_context(cb)
-        self.assertEqual(self.named_contexts, [])
-        yield gen.Wait('k1')
-        self.assertEqual(self.named_contexts, [])
-
-    @gen_test
-    def test_task_transfer_stack_context(self):
-        yield gen.Task(self.function_with_stack_context)
-        self.assertEqual(self.named_contexts, [])
-
-    def test_raise_after_stop(self):
-        # This pattern will be used in the following tests so make sure
-        # the exception propagates as expected.
-        @gen.engine
-        def f():
-            self.stop()
-            1 / 0
-
-        with self.assertRaises(ZeroDivisionError):
-            self.run_gen(f)
-
-    def test_sync_raise_return(self):
-        # gen.Return is allowed in @gen.engine, but it may not be used
-        # to return a value.
-        @gen.engine
-        def f():
-            self.stop(42)
-            raise gen.Return()
-
-        result = self.run_gen(f)
-        self.assertEqual(result, 42)
-
-    def test_async_raise_return(self):
-        @gen.engine
-        def f():
-            yield gen.Task(self.io_loop.add_callback)
-            self.stop(42)
-            raise gen.Return()
-
-        result = self.run_gen(f)
-        self.assertEqual(result, 42)
-
-    def test_sync_raise_return_value(self):
-        @gen.engine
-        def f():
-            raise gen.Return(42)
-
-        with self.assertRaises(gen.ReturnValueIgnoredError):
-            self.run_gen(f)
-
-    def test_sync_raise_return_value_tuple(self):
-        @gen.engine
-        def f():
-            raise gen.Return((1, 2))
-
-        with self.assertRaises(gen.ReturnValueIgnoredError):
-            self.run_gen(f)
-
-    def test_async_raise_return_value(self):
-        @gen.engine
-        def f():
-            yield gen.Task(self.io_loop.add_callback)
-            raise gen.Return(42)
-
-        with self.assertRaises(gen.ReturnValueIgnoredError):
-            self.run_gen(f)
-
-    def test_async_raise_return_value_tuple(self):
-        @gen.engine
-        def f():
-            yield gen.Task(self.io_loop.add_callback)
-            raise gen.Return((1, 2))
-
-        with self.assertRaises(gen.ReturnValueIgnoredError):
-            self.run_gen(f)
-
-    def test_return_value(self):
-        # It is an error to apply @gen.engine to a function that returns
-        # a value.
-        @gen.engine
-        def f():
-            return 42
-
-        with self.assertRaises(gen.ReturnValueIgnoredError):
-            self.run_gen(f)
-
-    def test_return_value_tuple(self):
-        # It is an error to apply @gen.engine to a function that returns
-        # a value.
-        @gen.engine
-        def f():
-            return (1, 2)
-
-        with self.assertRaises(gen.ReturnValueIgnoredError):
-            self.run_gen(f)
-
-    @skipNotCPython
-    def test_task_refcounting(self):
-        # On CPython, tasks and their arguments should be released immediately
-        # without waiting for garbage collection.
-        @gen.engine
-        def f():
-            class Foo(object):
-                pass
-            arg = Foo()
-            self.arg_ref = weakref.ref(arg)
-            task = gen.Task(self.io_loop.add_callback, arg=arg)
-            self.task_ref = weakref.ref(task)
-            yield task
-            self.stop()
-
-        self.run_gen(f)
-        self.assertIs(self.arg_ref(), None)
-        self.assertIs(self.task_ref(), None)
-
-
-class GenCoroutineTest(AsyncTestCase):
-    def setUp(self):
-        # Stray StopIteration exceptions can lead to tests exiting prematurely,
-        # so we need explicit checks here to make sure the tests run all
-        # the way through.
-        self.finished = False
-        super(GenCoroutineTest, self).setUp()
-
-    def tearDown(self):
-        super(GenCoroutineTest, self).tearDown()
-        assert self.finished
-
-    def test_attributes(self):
-        self.finished = True
-
-        def f():
-            yield gen.moment
-
-        coro = gen.coroutine(f)
-        self.assertEqual(coro.__name__, f.__name__)
-        self.assertEqual(coro.__module__, f.__module__)
-        self.assertIs(coro.__wrapped__, f)
-
-    def test_is_coroutine_function(self):
-        self.finished = True
-
-        def f():
-            yield gen.moment
-
-        coro = gen.coroutine(f)
-        self.assertFalse(gen.is_coroutine_function(f))
-        self.assertTrue(gen.is_coroutine_function(coro))
-        self.assertFalse(gen.is_coroutine_function(coro()))
-
-    @gen_test
-    def test_sync_gen_return(self):
-        @gen.coroutine
-        def f():
-            raise gen.Return(42)
-        result = yield f()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @gen_test
-    def test_async_gen_return(self):
-        @gen.coroutine
-        def f():
-            yield gen.Task(self.io_loop.add_callback)
-            raise gen.Return(42)
-        result = yield f()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @gen_test
-    def test_sync_return(self):
-        @gen.coroutine
-        def f():
-            return 42
-        result = yield f()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @skipBefore33
-    @gen_test
-    def test_async_return(self):
-        namespace = exec_test(globals(), locals(), """
-        @gen.coroutine
-        def f():
-            yield gen.Task(self.io_loop.add_callback)
-            return 42
-        """)
-        result = yield namespace['f']()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @skipBefore33
-    @gen_test
-    def test_async_early_return(self):
-        # A yield statement exists but is not executed, which means
-        # this function "returns" via an exception.  This exception
-        # doesn't happen before the exception handling is set up.
-        namespace = exec_test(globals(), locals(), """
-        @gen.coroutine
-        def f():
-            if True:
-                return 42
-            yield gen.Task(self.io_loop.add_callback)
-        """)
-        result = yield namespace['f']()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @skipBefore35
-    @gen_test
-    def test_async_await(self):
-        # This test verifies that an async function can await a
-        # yield-based gen.coroutine, and that a gen.coroutine
-        # (the test method itself) can yield an async function.
-        namespace = exec_test(globals(), locals(), """
-        async def f():
-            await gen.Task(self.io_loop.add_callback)
-            return 42
-        """)
-        result = yield namespace['f']()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @skipBefore35
-    @gen_test
-    def test_asyncio_sleep_zero(self):
-        # asyncio.sleep(0) turns into a special case (equivalent to
-        # `yield None`)
-        namespace = exec_test(globals(), locals(), """
-        async def f():
-            import asyncio
-            await asyncio.sleep(0)
-            return 42
-        """)
-        result = yield namespace['f']()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @skipBefore35
-    @gen_test
-    def test_async_await_mixed_multi_native_future(self):
-        namespace = exec_test(globals(), locals(), """
-        async def f1():
-            await gen.Task(self.io_loop.add_callback)
-            return 42
-        """)
-
-        @gen.coroutine
-        def f2():
-            yield gen.Task(self.io_loop.add_callback)
-            raise gen.Return(43)
-
-        results = yield [namespace['f1'](), f2()]
-        self.assertEqual(results, [42, 43])
-        self.finished = True
-
-    @skipBefore35
-    @gen_test
-    def test_async_await_mixed_multi_native_yieldpoint(self):
-        namespace = exec_test(globals(), locals(), """
-        async def f1():
-            await gen.Task(self.io_loop.add_callback)
-            return 42
-        """)
-
-        @gen.coroutine
-        def f2():
-            yield gen.Task(self.io_loop.add_callback)
-            raise gen.Return(43)
-
-        f2(callback=(yield gen.Callback('cb')))
-        results = yield [namespace['f1'](), gen.Wait('cb')]
-        self.assertEqual(results, [42, 43])
-        self.finished = True
-
-    @skipBefore35
-    @gen_test
-    def test_async_with_timeout(self):
-        namespace = exec_test(globals(), locals(), """
-        async def f1():
-            return 42
-        """)
-
-        result = yield gen.with_timeout(datetime.timedelta(hours=1),
-                                        namespace['f1']())
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @gen_test
-    def test_sync_return_no_value(self):
-        @gen.coroutine
-        def f():
-            return
-        result = yield f()
-        self.assertEqual(result, None)
-        self.finished = True
-
-    @gen_test
-    def test_async_return_no_value(self):
-        # Without a return value we don't need python 3.3.
-        @gen.coroutine
-        def f():
-            yield gen.Task(self.io_loop.add_callback)
-            return
-        result = yield f()
-        self.assertEqual(result, None)
-        self.finished = True
-
-    @gen_test
-    def test_sync_raise(self):
-        @gen.coroutine
-        def f():
-            1 / 0
-        # The exception is raised when the future is yielded
-        # (or equivalently when its result method is called),
-        # not when the function itself is called).
-        future = f()
-        with self.assertRaises(ZeroDivisionError):
-            yield future
-        self.finished = True
-
-    @gen_test
-    def test_async_raise(self):
-        @gen.coroutine
-        def f():
-            yield gen.Task(self.io_loop.add_callback)
-            1 / 0
-        future = f()
-        with self.assertRaises(ZeroDivisionError):
-            yield future
-        self.finished = True
-
-    @gen_test
-    def test_pass_callback(self):
-        @gen.coroutine
-        def f():
-            raise gen.Return(42)
-        result = yield gen.Task(f)
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @gen_test
-    def test_replace_yieldpoint_exception(self):
-        # Test exception handling: a coroutine can catch one exception
-        # raised by a yield point and raise a different one.
-        @gen.coroutine
-        def f1():
-            1 / 0
-
-        @gen.coroutine
-        def f2():
-            try:
-                yield f1()
-            except ZeroDivisionError:
-                raise KeyError()
-
-        future = f2()
-        with self.assertRaises(KeyError):
-            yield future
-        self.finished = True
-
-    @gen_test
-    def test_swallow_yieldpoint_exception(self):
-        # Test exception handling: a coroutine can catch an exception
-        # raised by a yield point and not raise a different one.
-        @gen.coroutine
-        def f1():
-            1 / 0
-
-        @gen.coroutine
-        def f2():
-            try:
-                yield f1()
-            except ZeroDivisionError:
-                raise gen.Return(42)
-
-        result = yield f2()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @gen_test
-    def test_replace_context_exception(self):
-        # Test exception handling: exceptions thrown into the stack context
-        # can be caught and replaced.
-        # Note that this test and the following are for behavior that is
-        # not really supported any more:  coroutines no longer create a
-        # stack context automatically; but one is created after the first
-        # YieldPoint (i.e. not a Future).
-        @gen.coroutine
-        def f2():
-            (yield gen.Callback(1))()
-            yield gen.Wait(1)
-            self.io_loop.add_callback(lambda: 1 / 0)
-            try:
-                yield gen.Task(self.io_loop.add_timeout,
-                               self.io_loop.time() + 10)
-            except ZeroDivisionError:
-                raise KeyError()
-
-        future = f2()
-        with self.assertRaises(KeyError):
-            yield future
-        self.finished = True
-
-    @gen_test
-    def test_swallow_context_exception(self):
-        # Test exception handling: exceptions thrown into the stack context
-        # can be caught and ignored.
-        @gen.coroutine
-        def f2():
-            (yield gen.Callback(1))()
-            yield gen.Wait(1)
-            self.io_loop.add_callback(lambda: 1 / 0)
-            try:
-                yield gen.Task(self.io_loop.add_timeout,
-                               self.io_loop.time() + 10)
-            except ZeroDivisionError:
-                raise gen.Return(42)
-
-        result = yield f2()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @gen_test
-    def test_moment(self):
-        calls = []
-
-        @gen.coroutine
-        def f(name, yieldable):
-            for i in range(5):
-                calls.append(name)
-                yield yieldable
-        # First, confirm the behavior without moment: each coroutine
-        # monopolizes the event loop until it finishes.
-        immediate = Future()
-        immediate.set_result(None)
-        yield [f('a', immediate), f('b', immediate)]
-        self.assertEqual(''.join(calls), 'aaaaabbbbb')
-
-        # With moment, they take turns.
-        calls = []
-        yield [f('a', gen.moment), f('b', gen.moment)]
-        self.assertEqual(''.join(calls), 'ababababab')
-        self.finished = True
-
-        calls = []
-        yield [f('a', gen.moment), f('b', immediate)]
-        self.assertEqual(''.join(calls), 'abbbbbaaaa')
-
-    @gen_test
-    def test_sleep(self):
-        yield gen.sleep(0.01)
-        self.finished = True
-
-    @skipBefore33
-    @gen_test
-    def test_py3_leak_exception_context(self):
-        class LeakedException(Exception):
-            pass
-
-        @gen.coroutine
-        def inner(iteration):
-            raise LeakedException(iteration)
-
-        try:
-            yield inner(1)
-        except LeakedException as e:
-            self.assertEqual(str(e), "1")
-            self.assertIsNone(e.__context__)
-
-        try:
-            yield inner(2)
-        except LeakedException as e:
-            self.assertEqual(str(e), "2")
-            self.assertIsNone(e.__context__)
-
-        self.finished = True
-
-    @skipNotCPython
-    def test_coroutine_refcounting(self):
-        # On CPython, tasks and their arguments should be released immediately
-        # without waiting for garbage collection.
-        @gen.coroutine
-        def inner():
-            class Foo(object):
-                pass
-            local_var = Foo()
-            self.local_ref = weakref.ref(local_var)
-            yield gen.coroutine(lambda: None)()
-            raise ValueError('Some error')
-
-        @gen.coroutine
-        def inner2():
-            try:
-                yield inner()
-            except ValueError:
-                pass
-
-        self.io_loop.run_sync(inner2, timeout=3)
-
-        self.assertIs(self.local_ref(), None)
-        self.finished = True
-
-
-class GenSequenceHandler(RequestHandler):
-    @asynchronous
-    @gen.engine
-    def get(self):
-        self.io_loop = self.request.connection.stream.io_loop
-        self.io_loop.add_callback((yield gen.Callback("k1")))
-        yield gen.Wait("k1")
-        self.write("1")
-        self.io_loop.add_callback((yield gen.Callback("k2")))
-        yield gen.Wait("k2")
-        self.write("2")
-        # reuse an old key
-        self.io_loop.add_callback((yield gen.Callback("k1")))
-        yield gen.Wait("k1")
-        self.finish("3")
-
-
-class GenCoroutineSequenceHandler(RequestHandler):
-    @gen.coroutine
-    def get(self):
-        self.io_loop = self.request.connection.stream.io_loop
-        self.io_loop.add_callback((yield gen.Callback("k1")))
-        yield gen.Wait("k1")
-        self.write("1")
-        self.io_loop.add_callback((yield gen.Callback("k2")))
-        yield gen.Wait("k2")
-        self.write("2")
-        # reuse an old key
-        self.io_loop.add_callback((yield gen.Callback("k1")))
-        yield gen.Wait("k1")
-        self.finish("3")
-
-
-class GenCoroutineUnfinishedSequenceHandler(RequestHandler):
-    @asynchronous
-    @gen.coroutine
-    def get(self):
-        self.io_loop = self.request.connection.stream.io_loop
-        self.io_loop.add_callback((yield gen.Callback("k1")))
-        yield gen.Wait("k1")
-        self.write("1")
-        self.io_loop.add_callback((yield gen.Callback("k2")))
-        yield gen.Wait("k2")
-        self.write("2")
-        # reuse an old key
-        self.io_loop.add_callback((yield gen.Callback("k1")))
-        yield gen.Wait("k1")
-        # just write, don't finish
-        self.write("3")
-
-
-class GenTaskHandler(RequestHandler):
-    @asynchronous
-    @gen.engine
-    def get(self):
-        io_loop = self.request.connection.stream.io_loop
-        client = AsyncHTTPClient(io_loop=io_loop)
-        response = yield gen.Task(client.fetch, self.get_argument('url'))
-        response.rethrow()
-        self.finish(b"got response: " + response.body)
-
-
-class GenExceptionHandler(RequestHandler):
-    @asynchronous
-    @gen.engine
-    def get(self):
-        # This test depends on the order of the two decorators.
-        io_loop = self.request.connection.stream.io_loop
-        yield gen.Task(io_loop.add_callback)
-        raise Exception("oops")
-
-
-class GenCoroutineExceptionHandler(RequestHandler):
-    @gen.coroutine
-    def get(self):
-        # This test depends on the order of the two decorators.
-        io_loop = self.request.connection.stream.io_loop
-        yield gen.Task(io_loop.add_callback)
-        raise Exception("oops")
-
-
-class GenYieldExceptionHandler(RequestHandler):
-    @asynchronous
-    @gen.engine
-    def get(self):
-        io_loop = self.request.connection.stream.io_loop
-        # Test the interaction of the two stack_contexts.
-
-        def fail_task(callback):
-            io_loop.add_callback(lambda: 1 / 0)
-        try:
-            yield gen.Task(fail_task)
-            raise Exception("did not get expected exception")
-        except ZeroDivisionError:
-            self.finish('ok')
-
-
-# "Undecorated" here refers to the absence of @asynchronous.
-class UndecoratedCoroutinesHandler(RequestHandler):
-    @gen.coroutine
-    def prepare(self):
-        self.chunks = []
-        yield gen.Task(IOLoop.current().add_callback)
-        self.chunks.append('1')
-
-    @gen.coroutine
-    def get(self):
-        self.chunks.append('2')
-        yield gen.Task(IOLoop.current().add_callback)
-        self.chunks.append('3')
-        yield gen.Task(IOLoop.current().add_callback)
-        self.write(''.join(self.chunks))
-
-
-class AsyncPrepareErrorHandler(RequestHandler):
-    @gen.coroutine
-    def prepare(self):
-        yield gen.Task(IOLoop.current().add_callback)
-        raise HTTPError(403)
-
-    def get(self):
-        self.finish('ok')
-
-
-class NativeCoroutineHandler(RequestHandler):
-    if sys.version_info > (3, 5):
-        exec(textwrap.dedent("""
-        async def get(self):
-            await gen.Task(IOLoop.current().add_callback)
-            self.write("ok")
-        """))
-
-
-class GenWebTest(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application([
-            ('/sequence', GenSequenceHandler),
-            ('/coroutine_sequence', GenCoroutineSequenceHandler),
-            ('/coroutine_unfinished_sequence',
-             GenCoroutineUnfinishedSequenceHandler),
-            ('/task', GenTaskHandler),
-            ('/exception', GenExceptionHandler),
-            ('/coroutine_exception', GenCoroutineExceptionHandler),
-            ('/yield_exception', GenYieldExceptionHandler),
-            ('/undecorated_coroutine', UndecoratedCoroutinesHandler),
-            ('/async_prepare_error', AsyncPrepareErrorHandler),
-            ('/native_coroutine', NativeCoroutineHandler),
-        ])
-
-    def test_sequence_handler(self):
-        response = self.fetch('/sequence')
-        self.assertEqual(response.body, b"123")
-
-    def test_coroutine_sequence_handler(self):
-        response = self.fetch('/coroutine_sequence')
-        self.assertEqual(response.body, b"123")
-
-    def test_coroutine_unfinished_sequence_handler(self):
-        response = self.fetch('/coroutine_unfinished_sequence')
-        self.assertEqual(response.body, b"123")
-
-    def test_task_handler(self):
-        response = self.fetch('/task?url=%s' % url_escape(self.get_url('/sequence')))
-        self.assertEqual(response.body, b"got response: 123")
-
-    def test_exception_handler(self):
-        # Make sure we get an error and not a timeout
-        with ExpectLog(app_log, "Uncaught exception GET /exception"):
-            response = self.fetch('/exception')
-        self.assertEqual(500, response.code)
-
-    def test_coroutine_exception_handler(self):
-        # Make sure we get an error and not a timeout
-        with ExpectLog(app_log, "Uncaught exception GET /coroutine_exception"):
-            response = self.fetch('/coroutine_exception')
-        self.assertEqual(500, response.code)
-
-    def test_yield_exception_handler(self):
-        response = self.fetch('/yield_exception')
-        self.assertEqual(response.body, b'ok')
-
-    def test_undecorated_coroutines(self):
-        response = self.fetch('/undecorated_coroutine')
-        self.assertEqual(response.body, b'123')
-
-    def test_async_prepare_error_handler(self):
-        response = self.fetch('/async_prepare_error')
-        self.assertEqual(response.code, 403)
-
-    @skipBefore35
-    def test_native_coroutine_handler(self):
-        response = self.fetch('/native_coroutine')
-        self.assertEqual(response.code, 200)
-        self.assertEqual(response.body, b'ok')
-
-
-class WithTimeoutTest(AsyncTestCase):
-    @gen_test
-    def test_timeout(self):
-        with self.assertRaises(gen.TimeoutError):
-            yield gen.with_timeout(datetime.timedelta(seconds=0.1),
-                                   Future())
-
-    @gen_test
-    def test_completes_before_timeout(self):
-        future = Future()
-        self.io_loop.add_timeout(datetime.timedelta(seconds=0.1),
-                                 lambda: future.set_result('asdf'))
-        result = yield gen.with_timeout(datetime.timedelta(seconds=3600),
-                                        future, io_loop=self.io_loop)
-        self.assertEqual(result, 'asdf')
-
-    @gen_test
-    def test_fails_before_timeout(self):
-        future = Future()
-        self.io_loop.add_timeout(
-            datetime.timedelta(seconds=0.1),
-            lambda: future.set_exception(ZeroDivisionError()))
-        with self.assertRaises(ZeroDivisionError):
-            yield gen.with_timeout(datetime.timedelta(seconds=3600),
-                                   future, io_loop=self.io_loop)
-
-    @gen_test
-    def test_already_resolved(self):
-        future = Future()
-        future.set_result('asdf')
-        result = yield gen.with_timeout(datetime.timedelta(seconds=3600),
-                                        future, io_loop=self.io_loop)
-        self.assertEqual(result, 'asdf')
-
-    @unittest.skipIf(futures is None, 'futures module not present')
-    @gen_test
-    def test_timeout_concurrent_future(self):
-        with futures.ThreadPoolExecutor(1) as executor:
-            with self.assertRaises(gen.TimeoutError):
-                yield gen.with_timeout(self.io_loop.time(),
-                                       executor.submit(time.sleep, 0.1))
-
-    @unittest.skipIf(futures is None, 'futures module not present')
-    @gen_test
-    def test_completed_concurrent_future(self):
-        with futures.ThreadPoolExecutor(1) as executor:
-            yield gen.with_timeout(datetime.timedelta(seconds=3600),
-                                   executor.submit(lambda: None))
-
-
-class WaitIteratorTest(AsyncTestCase):
-    @gen_test
-    def test_empty_iterator(self):
-        g = gen.WaitIterator()
-        self.assertTrue(g.done(), 'empty generator iterated')
-
-        with self.assertRaises(ValueError):
-            g = gen.WaitIterator(False, bar=False)
-
-        self.assertEqual(g.current_index, None, "bad nil current index")
-        self.assertEqual(g.current_future, None, "bad nil current future")
-
-    @gen_test
-    def test_already_done(self):
-        f1 = Future()
-        f2 = Future()
-        f3 = Future()
-        f1.set_result(24)
-        f2.set_result(42)
-        f3.set_result(84)
-
-        g = gen.WaitIterator(f1, f2, f3)
-        i = 0
-        while not g.done():
-            r = yield g.next()
-            # Order is not guaranteed, but the current implementation
-            # preserves ordering of already-done Futures.
-            if i == 0:
-                self.assertEqual(g.current_index, 0)
-                self.assertIs(g.current_future, f1)
-                self.assertEqual(r, 24)
-            elif i == 1:
-                self.assertEqual(g.current_index, 1)
-                self.assertIs(g.current_future, f2)
-                self.assertEqual(r, 42)
-            elif i == 2:
-                self.assertEqual(g.current_index, 2)
-                self.assertIs(g.current_future, f3)
-                self.assertEqual(r, 84)
-            i += 1
-
-        self.assertEqual(g.current_index, None, "bad nil current index")
-        self.assertEqual(g.current_future, None, "bad nil current future")
-
-        dg = gen.WaitIterator(f1=f1, f2=f2)
-
-        while not dg.done():
-            dr = yield dg.next()
-            if dg.current_index == "f1":
-                self.assertTrue(dg.current_future == f1 and dr == 24,
-                                "WaitIterator dict status incorrect")
-            elif dg.current_index == "f2":
-                self.assertTrue(dg.current_future == f2 and dr == 42,
-                                "WaitIterator dict status incorrect")
-            else:
-                self.fail("got bad WaitIterator index {}".format(
-                    dg.current_index))
-
-            i += 1
-
-        self.assertEqual(dg.current_index, None, "bad nil current index")
-        self.assertEqual(dg.current_future, None, "bad nil current future")
-
-    def finish_coroutines(self, iteration, futures):
-        if iteration == 3:
-            futures[2].set_result(24)
-        elif iteration == 5:
-            futures[0].set_exception(ZeroDivisionError())
-        elif iteration == 8:
-            futures[1].set_result(42)
-            futures[3].set_result(84)
-
-        if iteration < 8:
-            self.io_loop.add_callback(self.finish_coroutines, iteration + 1, futures)
-
-    @gen_test
-    def test_iterator(self):
-        futures = [Future(), Future(), Future(), Future()]
-
-        self.finish_coroutines(0, futures)
-
-        g = gen.WaitIterator(*futures)
-
-        i = 0
-        while not g.done():
-            try:
-                r = yield g.next()
-            except ZeroDivisionError:
-                self.assertIs(g.current_future, futures[0],
-                              'exception future invalid')
-            else:
-                if i == 0:
-                    self.assertEqual(r, 24, 'iterator value incorrect')
-                    self.assertEqual(g.current_index, 2, 'wrong index')
-                elif i == 2:
-                    self.assertEqual(r, 42, 'iterator value incorrect')
-                    self.assertEqual(g.current_index, 1, 'wrong index')
-                elif i == 3:
-                    self.assertEqual(r, 84, 'iterator value incorrect')
-                    self.assertEqual(g.current_index, 3, 'wrong index')
-            i += 1
-
-    @skipBefore35
-    @gen_test
-    def test_iterator_async_await(self):
-        # Recreate the previous test with py35 syntax. It's a little clunky
-        # because of the way the previous test handles an exception on
-        # a single iteration.
-        futures = [Future(), Future(), Future(), Future()]
-        self.finish_coroutines(0, futures)
-        self.finished = False
-
-        namespace = exec_test(globals(), locals(), """
-        async def f():
-            i = 0
-            g = gen.WaitIterator(*futures)
-            try:
-                async for r in g:
-                    if i == 0:
-                        self.assertEqual(r, 24, 'iterator value incorrect')
-                        self.assertEqual(g.current_index, 2, 'wrong index')
-                    else:
-                        raise Exception("expected exception on iteration 1")
-                    i += 1
-            except ZeroDivisionError:
-                i += 1
-            async for r in g:
-                if i == 2:
-                    self.assertEqual(r, 42, 'iterator value incorrect')
-                    self.assertEqual(g.current_index, 1, 'wrong index')
-                elif i == 3:
-                    self.assertEqual(r, 84, 'iterator value incorrect')
-                    self.assertEqual(g.current_index, 3, 'wrong index')
-                else:
-                    raise Exception("didn't expect iteration %d" % i)
-                i += 1
-            self.finished = True
-        """)
-        yield namespace['f']()
-        self.assertTrue(self.finished)
-
-    @gen_test
-    def test_no_ref(self):
-        # In this usage, there is no direct hard reference to the
-        # WaitIterator itself, only the Future it returns. Since
-        # WaitIterator uses weak references internally to improve GC
-        # performance, this used to cause problems.
-        yield gen.with_timeout(datetime.timedelta(seconds=0.1),
-                               gen.WaitIterator(gen.sleep(0)).next())
-
-
-class RunnerGCTest(AsyncTestCase):
-    """Github issue 1769: Runner objects can get GCed unexpectedly"""
-    @gen_test
-    def test_gc(self):
-        """Runners shouldn't GC if future is alive"""
-        # Create the weakref
-        weakref_scope = [None]
-
-        def callback():
-            gc.collect(2)
-            weakref_scope[0]().set_result(123)
-
-        @gen.coroutine
-        def tester():
-            fut = Future()
-            weakref_scope[0] = weakref.ref(fut)
-            self.io_loop.add_callback(callback)
-            yield fut
-
-        yield gen.with_timeout(
-            datetime.timedelta(seconds=0.2),
-            tester()
-        )
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/salt/ext/tornado/test/gettext_translations/extract_me.py b/salt/ext/tornado/test/gettext_translations/extract_me.py
deleted file mode 100644
index 4bca943b5f3..00000000000
--- a/salt/ext/tornado/test/gettext_translations/extract_me.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# flake8: noqa
-# Dummy source file to allow creation of the initial .po file in the
-# same way as a real project.  I'm not entirely sure about the real
-# workflow here, but this seems to work.
-#
-# 1) xgettext --language=Python --keyword=_:1,2 --keyword=pgettext:1c,2 --keyword=pgettext:1c,2,3 extract_me.py -o tornado_test.po
-# 2) Edit tornado_test.po, setting CHARSET, Plural-Forms and setting msgstr
-# 3) msgfmt tornado_test.po -o tornado_test.mo
-# 4) Put the file in the proper location: $LANG/LC_MESSAGES
-# pylint: skip-file
-
-from __future__ import absolute_import, division, print_function
-_("school")
-pgettext("law", "right")
-pgettext("good", "right")
-pgettext("organization", "club", "clubs", 1)
-pgettext("stick", "club", "clubs", 1)
diff --git a/salt/ext/tornado/test/gettext_translations/fr_FR/LC_MESSAGES/tornado_test.po b/salt/ext/tornado/test/gettext_translations/fr_FR/LC_MESSAGES/tornado_test.po
deleted file mode 100644
index 88d72c8623a..00000000000
--- a/salt/ext/tornado/test/gettext_translations/fr_FR/LC_MESSAGES/tornado_test.po
+++ /dev/null
@@ -1,47 +0,0 @@
-# SOME DESCRIPTIVE TITLE.
-# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
-# This file is distributed under the same license as the PACKAGE package.
-# FIRST AUTHOR , YEAR.
-#
-#, fuzzy
-msgid ""
-msgstr ""
-"Project-Id-Version: PACKAGE VERSION\n"
-"Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2015-01-27 11:05+0300\n"
-"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
-"Last-Translator: FULL NAME \n"
-"Language-Team: LANGUAGE \n"
-"Language: \n"
-"MIME-Version: 1.0\n"
-"Content-Type: text/plain; charset=utf-8\n"
-"Content-Transfer-Encoding: 8bit\n"
-"Plural-Forms: nplurals=2; plural=(n > 1);\n"
-
-#: extract_me.py:11
-msgid "school"
-msgstr "école"
-
-#: extract_me.py:12
-msgctxt "law"
-msgid "right"
-msgstr "le droit"
-
-#: extract_me.py:13
-msgctxt "good"
-msgid "right"
-msgstr "le bien"
-
-#: extract_me.py:14
-msgctxt "organization"
-msgid "club"
-msgid_plural "clubs"
-msgstr[0] "le club"
-msgstr[1] "les clubs"
-
-#: extract_me.py:15
-msgctxt "stick"
-msgid "club"
-msgid_plural "clubs"
-msgstr[0] "le bâton"
-msgstr[1] "les bâtons"
diff --git a/salt/ext/tornado/test/http1connection_test.py b/salt/ext/tornado/test/http1connection_test.py
deleted file mode 100644
index 8ee26c433d3..00000000000
--- a/salt/ext/tornado/test/http1connection_test.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# pylint: skip-file
-from __future__ import absolute_import, division, print_function
-
-import socket
-
-from salt.ext.tornado.http1connection import HTTP1Connection
-from salt.ext.tornado.httputil import HTTPMessageDelegate
-from salt.ext.tornado.iostream import IOStream
-from salt.ext.tornado.locks import Event
-from salt.ext.tornado.netutil import add_accept_handler
-from salt.ext.tornado.testing import AsyncTestCase, bind_unused_port, gen_test
-
-
-class HTTP1ConnectionTest(AsyncTestCase):
-    def setUp(self):
-        super(HTTP1ConnectionTest, self).setUp()
-        self.asyncSetUp()
-
-    @gen_test
-    def asyncSetUp(self):
-        listener, port = bind_unused_port()
-        event = Event()
-
-        def accept_callback(conn, addr):
-            self.server_stream = IOStream(conn)
-            self.addCleanup(self.server_stream.close)
-            event.set()
-
-        add_accept_handler(listener, accept_callback)
-        self.client_stream = IOStream(socket.socket())
-        self.addCleanup(self.client_stream.close)
-        yield [self.client_stream.connect(('127.0.0.1', port)),
-               event.wait()]
-        self.io_loop.remove_handler(listener)
-        listener.close()
-
-    @gen_test
-    def test_http10_no_content_length(self):
-        # Regression test for a bug in which can_keep_alive would crash
-        # for an HTTP/1.0 (not 1.1) response with no content-length.
-        conn = HTTP1Connection(self.client_stream, True)
-        self.server_stream.write(b"HTTP/1.0 200 Not Modified\r\n\r\nhello")
-        self.server_stream.close()
-
-        event = Event()
-        test = self
-        body = []
-
-        class Delegate(HTTPMessageDelegate):
-            def headers_received(self, start_line, headers):
-                test.code = start_line.code
-
-            def data_received(self, data):
-                body.append(data)
-
-            def finish(self):
-                event.set()
-
-        yield conn.read_response(Delegate())
-        yield event.wait()
-        self.assertEqual(self.code, 200)
-        self.assertEqual(b''.join(body), b'hello')
diff --git a/salt/ext/tornado/test/httpclient_test.py b/salt/ext/tornado/test/httpclient_test.py
deleted file mode 100644
index 9aa17c43464..00000000000
--- a/salt/ext/tornado/test/httpclient_test.py
+++ /dev/null
@@ -1,686 +0,0 @@
-#!/usr/bin/env python
-# pylint: skip-file
-
-from __future__ import absolute_import, division, print_function
-
-import base64
-import binascii
-from contextlib import closing
-import copy
-import functools
-import sys
-import threading
-import datetime
-from io import BytesIO
-
-from salt.ext.tornado.escape import utf8, native_str
-from salt.ext.tornado import gen
-from salt.ext.tornado.httpclient import HTTPRequest, HTTPResponse, _RequestProxy, HTTPError, HTTPClient
-from salt.ext.tornado.httpserver import HTTPServer
-from salt.ext.tornado.ioloop import IOLoop
-from salt.ext.tornado.iostream import IOStream
-from salt.ext.tornado.log import gen_log
-from salt.ext.tornado import netutil
-from salt.ext.tornado.stack_context import ExceptionStackContext, NullContext
-from salt.ext.tornado.testing import AsyncHTTPTestCase, bind_unused_port, gen_test, ExpectLog
-from salt.ext.tornado.test.util import unittest, skipOnTravis
-from salt.ext.tornado.web import Application, RequestHandler, url
-from salt.ext.tornado.httputil import format_timestamp, HTTPHeaders
-
-
-class HelloWorldHandler(RequestHandler):
-    def get(self):
-        name = self.get_argument("name", "world")
-        self.set_header("Content-Type", "text/plain")
-        self.finish("Hello %s!" % name)
-
-
-class PostHandler(RequestHandler):
-    def post(self):
-        self.finish("Post arg1: %s, arg2: %s" % (
-            self.get_argument("arg1"), self.get_argument("arg2")))
-
-
-class PutHandler(RequestHandler):
-    def put(self):
-        self.write("Put body: ")
-        self.write(self.request.body)
-
-
-class RedirectHandler(RequestHandler):
-    def prepare(self):
-        self.write('redirects can have bodies too')
-        self.redirect(self.get_argument("url"),
-                      status=int(self.get_argument("status", "302")))
-
-
-class ChunkHandler(RequestHandler):
-    @gen.coroutine
-    def get(self):
-        self.write("asdf")
-        self.flush()
-        # Wait a bit to ensure the chunks are sent and received separately.
-        yield gen.sleep(0.01)
-        self.write("qwer")
-
-
-class AuthHandler(RequestHandler):
-    def get(self):
-        self.finish(self.request.headers["Authorization"])
-
-
-class CountdownHandler(RequestHandler):
-    def get(self, count):
-        count = int(count)
-        if count > 0:
-            self.redirect(self.reverse_url("countdown", count - 1))
-        else:
-            self.write("Zero")
-
-
-class EchoPostHandler(RequestHandler):
-    def post(self):
-        self.write(self.request.body)
-
-
-class UserAgentHandler(RequestHandler):
-    def get(self):
-        self.write(self.request.headers.get('User-Agent', 'User agent not set'))
-
-
-class ContentLength304Handler(RequestHandler):
-    def get(self):
-        self.set_status(304)
-        self.set_header('Content-Length', 42)
-
-    def _clear_headers_for_304(self):
-        # Tornado strips content-length from 304 responses, but here we
-        # want to simulate servers that include the headers anyway.
-        pass
-
-
-class PatchHandler(RequestHandler):
-
-    def patch(self):
-        "Return the request payload - so we can check it is being kept"
-        self.write(self.request.body)
-
-
-class AllMethodsHandler(RequestHandler):
-    SUPPORTED_METHODS = RequestHandler.SUPPORTED_METHODS + ('OTHER',)
-
-    def method(self):
-        self.write(self.request.method)
-
-    get = post = put = delete = options = patch = other = method
-
-
-class SetHeaderHandler(RequestHandler):
-    def get(self):
-        # Use get_arguments for keys to get strings, but
-        # request.arguments for values to get bytes.
-        for k, v in zip(self.get_arguments('k'),
-                        self.request.arguments['v']):
-            self.set_header(k, v)
-
-# These tests end up getting run redundantly: once here with the default
-# HTTPClient implementation, and then again in each implementation's own
-# test suite.
-
-
-class HTTPClientCommonTestCase(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application([
-            url("/hello", HelloWorldHandler),
-            url("/post", PostHandler),
-            url("/put", PutHandler),
-            url("/redirect", RedirectHandler),
-            url("/chunk", ChunkHandler),
-            url("/auth", AuthHandler),
-            url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
-            url("/echopost", EchoPostHandler),
-            url("/user_agent", UserAgentHandler),
-            url("/304_with_content_length", ContentLength304Handler),
-            url("/all_methods", AllMethodsHandler),
-            url('/patch', PatchHandler),
-            url('/set_header', SetHeaderHandler),
-        ], gzip=True)
-
-    def test_patch_receives_payload(self):
-        body = b"some patch data"
-        response = self.fetch("/patch", method='PATCH', body=body)
-        self.assertEqual(response.code, 200)
-        self.assertEqual(response.body, body)
-
-    @skipOnTravis
-    def test_hello_world(self):
-        response = self.fetch("/hello")
-        self.assertEqual(response.code, 200)
-        self.assertEqual(response.headers["Content-Type"], "text/plain")
-        self.assertEqual(response.body, b"Hello world!")
-        self.assertEqual(int(response.request_time), 0)
-
-        response = self.fetch("/hello?name=Ben")
-        self.assertEqual(response.body, b"Hello Ben!")
-
-    def test_streaming_callback(self):
-        # streaming_callback is also tested in test_chunked
-        chunks = []
-        response = self.fetch("/hello",
-                              streaming_callback=chunks.append)
-        # with streaming_callback, data goes to the callback and not response.body
-        self.assertEqual(chunks, [b"Hello world!"])
-        self.assertFalse(response.body)
-
-    def test_post(self):
-        response = self.fetch("/post", method="POST",
-                              body="arg1=foo&arg2=bar")
-        self.assertEqual(response.code, 200)
-        self.assertEqual(response.body, b"Post arg1: foo, arg2: bar")
-
-    def test_chunked(self):
-        response = self.fetch("/chunk")
-        self.assertEqual(response.body, b"asdfqwer")
-
-        chunks = []
-        response = self.fetch("/chunk",
-                              streaming_callback=chunks.append)
-        self.assertEqual(chunks, [b"asdf", b"qwer"])
-        self.assertFalse(response.body)
-
-    def test_chunked_close(self):
-        # test case in which chunks spread read-callback processing
-        # over several ioloop iterations, but the connection is already closed.
-        sock, port = bind_unused_port()
-        with closing(sock):
-            def write_response(stream, request_data):
-                if b"HTTP/1." not in request_data:
-                    self.skipTest("requires HTTP/1.x")
-                stream.write(b"""\
-HTTP/1.1 200 OK
-Transfer-Encoding: chunked
-
-1
-1
-1
-2
-0
-
-""".replace(b"\n", b"\r\n"), callback=stream.close)
-
-            def accept_callback(conn, address):
-                # fake an HTTP server using chunked encoding where the final chunks
-                # and connection close all happen at once
-                stream = IOStream(conn, io_loop=self.io_loop)
-                stream.read_until(b"\r\n\r\n",
-                                  functools.partial(write_response, stream))
-            netutil.add_accept_handler(sock, accept_callback, self.io_loop)
-            self.http_client.fetch("http://127.0.0.1:%d/" % port, self.stop)
-            resp = self.wait()
-            resp.rethrow()
-            self.assertEqual(resp.body, b"12")
-            self.io_loop.remove_handler(sock.fileno())
-
-    def test_streaming_stack_context(self):
-        chunks = []
-        exc_info = []
-
-        def error_handler(typ, value, tb):
-            exc_info.append((typ, value, tb))
-            return True
-
-        def streaming_cb(chunk):
-            chunks.append(chunk)
-            if chunk == b'qwer':
-                1 / 0
-
-        with ExceptionStackContext(error_handler):
-            self.fetch('/chunk', streaming_callback=streaming_cb)
-
-        self.assertEqual(chunks, [b'asdf', b'qwer'])
-        self.assertEqual(1, len(exc_info))
-        self.assertIs(exc_info[0][0], ZeroDivisionError)
-
-    def test_basic_auth(self):
-        self.assertEqual(self.fetch("/auth", auth_username="Aladdin",
-                                    auth_password="open sesame").body,
-                         b"Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==")
-
-    def test_basic_auth_explicit_mode(self):
-        self.assertEqual(self.fetch("/auth", auth_username="Aladdin",
-                                    auth_password="open sesame",
-                                    auth_mode="basic").body,
-                         b"Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==")
-
-    def test_unsupported_auth_mode(self):
-        # curl and simple clients handle errors a bit differently; the
-        # important thing is that they don't fall back to basic auth
-        # on an unknown mode.
-        with ExpectLog(gen_log, "uncaught exception", required=False):
-            with self.assertRaises((ValueError, HTTPError)):
-                response = self.fetch("/auth", auth_username="Aladdin",
-                                      auth_password="open sesame",
-                                      auth_mode="asdf")
-                response.rethrow()
-
-    def test_follow_redirect(self):
-        response = self.fetch("/countdown/2", follow_redirects=False)
-        self.assertEqual(302, response.code)
-        self.assertTrue(response.headers["Location"].endswith("/countdown/1"))
-
-        response = self.fetch("/countdown/2")
-        self.assertEqual(200, response.code)
-        self.assertTrue(response.effective_url.endswith("/countdown/0"))
-        self.assertEqual(b"Zero", response.body)
-
-    def test_credentials_in_url(self):
-        url = self.get_url("/auth").replace("http://", "http://me:secret@")
-        self.http_client.fetch(url, self.stop)
-        response = self.wait()
-        self.assertEqual(b"Basic " + base64.b64encode(b"me:secret"),
-                         response.body)
-
-    def test_body_encoding(self):
-        unicode_body = u"\xe9"
-        byte_body = binascii.a2b_hex(b"e9")
-
-        # unicode string in body gets converted to utf8
-        response = self.fetch("/echopost", method="POST", body=unicode_body,
-                              headers={"Content-Type": "application/blah"})
-        self.assertEqual(response.headers["Content-Length"], "2")
-        self.assertEqual(response.body, utf8(unicode_body))
-
-        # byte strings pass through directly
-        response = self.fetch("/echopost", method="POST",
-                              body=byte_body,
-                              headers={"Content-Type": "application/blah"})
-        self.assertEqual(response.headers["Content-Length"], "1")
-        self.assertEqual(response.body, byte_body)
-
-        # Mixing unicode in headers and byte string bodies shouldn't
-        # break anything
-        response = self.fetch("/echopost", method="POST", body=byte_body,
-                              headers={"Content-Type": "application/blah"},
-                              user_agent=u"foo")
-        self.assertEqual(response.headers["Content-Length"], "1")
-        self.assertEqual(response.body, byte_body)
-
-    def test_types(self):
-        response = self.fetch("/hello")
-        self.assertEqual(type(response.body), bytes)
-        self.assertEqual(type(response.headers["Content-Type"]), str)
-        self.assertEqual(type(response.code), int)
-        self.assertEqual(type(response.effective_url), str)
-
-    def test_header_callback(self):
-        first_line = []
-        headers = {}
-        chunks = []
-
-        def header_callback(header_line):
-            if header_line.startswith('HTTP/1.1 101'):
-                # Upgrading to HTTP/2
-                pass
-            elif header_line.startswith('HTTP/'):
-                first_line.append(header_line)
-            elif header_line != '\r\n':
-                k, v = header_line.split(':', 1)
-                headers[k.lower()] = v.strip()
-
-        def streaming_callback(chunk):
-            # All header callbacks are run before any streaming callbacks,
-            # so the header data is available to process the data as it
-            # comes in.
-            self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8')
-            chunks.append(chunk)
-
-        self.fetch('/chunk', header_callback=header_callback,
-                   streaming_callback=streaming_callback)
-        self.assertEqual(len(first_line), 1, first_line)
-        self.assertRegexpMatches(first_line[0], 'HTTP/[0-9]\\.[0-9] 200.*\r\n')
-        self.assertEqual(chunks, [b'asdf', b'qwer'])
-
-    def test_header_callback_stack_context(self):
-        exc_info = []
-
-        def error_handler(typ, value, tb):
-            exc_info.append((typ, value, tb))
-            return True
-
-        def header_callback(header_line):
-            if header_line.lower().startswith('content-type:'):
-                1 / 0
-
-        with ExceptionStackContext(error_handler):
-            self.fetch('/chunk', header_callback=header_callback)
-        self.assertEqual(len(exc_info), 1)
-        self.assertIs(exc_info[0][0], ZeroDivisionError)
-
-    def test_configure_defaults(self):
-        defaults = dict(user_agent='TestDefaultUserAgent', allow_ipv6=False)
-        # Construct a new instance of the configured client class
-        client = self.http_client.__class__(self.io_loop, force_instance=True,
-                                            defaults=defaults)
-        try:
-            client.fetch(self.get_url('/user_agent'), callback=self.stop)
-            response = self.wait()
-            self.assertEqual(response.body, b'TestDefaultUserAgent')
-        finally:
-            client.close()
-
-    def test_header_types(self):
-        # Header values may be passed as character or utf8 byte strings,
-        # in a plain dictionary or an HTTPHeaders object.
-        # Keys must always be the native str type.
-        # All combinations should have the same results on the wire.
-        for value in [u"MyUserAgent", b"MyUserAgent"]:
-            for container in [dict, HTTPHeaders]:
-                headers = container()
-                headers['User-Agent'] = value
-                resp = self.fetch('/user_agent', headers=headers)
-                self.assertEqual(
-                    resp.body, b"MyUserAgent",
-                    "response=%r, value=%r, container=%r" %
-                    (resp.body, value, container))
-
-    def test_multi_line_headers(self):
-        # Multi-line http headers are rare but rfc-allowed
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
-        sock, port = bind_unused_port()
-        with closing(sock):
-            def write_response(stream, request_data):
-                if b"HTTP/1." not in request_data:
-                    self.skipTest("requires HTTP/1.x")
-                stream.write(b"""\
-HTTP/1.1 200 OK
-X-XSS-Protection: 1;
-\tmode=block
-
-""".replace(b"\n", b"\r\n"), callback=stream.close)
-
-            def accept_callback(conn, address):
-                stream = IOStream(conn, io_loop=self.io_loop)
-                stream.read_until(b"\r\n\r\n",
-                                  functools.partial(write_response, stream))
-            netutil.add_accept_handler(sock, accept_callback, self.io_loop)
-            self.http_client.fetch("http://127.0.0.1:%d/" % port, self.stop)
-            resp = self.wait()
-            resp.rethrow()
-            self.assertEqual(resp.headers['X-XSS-Protection'], "1; mode=block")
-            self.io_loop.remove_handler(sock.fileno())
-
-    def test_304_with_content_length(self):
-        # According to the spec 304 responses SHOULD NOT include
-        # Content-Length or other entity headers, but some servers do it
-        # anyway.
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5
-        response = self.fetch('/304_with_content_length')
-        self.assertEqual(response.code, 304)
-        self.assertEqual(response.headers['Content-Length'], '42')
-
-    def test_final_callback_stack_context(self):
-        # The final callback should be run outside of the httpclient's
-        # stack_context.  We want to ensure that there is not stack_context
-        # between the user's callback and the IOLoop, so monkey-patch
-        # IOLoop.handle_callback_exception and disable the test harness's
-        # context with a NullContext.
-        # Note that this does not apply to secondary callbacks (header
-        # and streaming_callback), as errors there must be seen as errors
-        # by the http client so it can clean up the connection.
-        exc_info = []
-
-        def handle_callback_exception(callback):
-            exc_info.append(sys.exc_info())
-            self.stop()
-        self.io_loop.handle_callback_exception = handle_callback_exception
-        with NullContext():
-            self.http_client.fetch(self.get_url('/hello'),
-                                   lambda response: 1 / 0)
-        self.wait()
-        self.assertEqual(exc_info[0][0], ZeroDivisionError)
-
-    @gen_test
-    def test_future_interface(self):
-        response = yield self.http_client.fetch(self.get_url('/hello'))
-        self.assertEqual(response.body, b'Hello world!')
-
-    @gen_test
-    def test_future_http_error(self):
-        with self.assertRaises(HTTPError) as context:
-            yield self.http_client.fetch(self.get_url('/notfound'))
-        self.assertEqual(context.exception.code, 404)
-        self.assertEqual(context.exception.response.code, 404)
-
-    @gen_test
-    def test_future_http_error_no_raise(self):
-        response = yield self.http_client.fetch(self.get_url('/notfound'), raise_error=False)
-        self.assertEqual(response.code, 404)
-
-    @gen_test
-    def test_reuse_request_from_response(self):
-        # The response.request attribute should be an HTTPRequest, not
-        # a _RequestProxy.
-        # This test uses self.http_client.fetch because self.fetch calls
-        # self.get_url on the input unconditionally.
-        url = self.get_url('/hello')
-        response = yield self.http_client.fetch(url)
-        self.assertEqual(response.request.url, url)
-        self.assertTrue(isinstance(response.request, HTTPRequest))
-        response2 = yield self.http_client.fetch(response.request)
-        self.assertEqual(response2.body, b'Hello world!')
-
-    def test_all_methods(self):
-        for method in ['GET', 'DELETE', 'OPTIONS']:
-            response = self.fetch('/all_methods', method=method)
-            self.assertEqual(response.body, utf8(method))
-        for method in ['POST', 'PUT', 'PATCH']:
-            response = self.fetch('/all_methods', method=method, body=b'')
-            self.assertEqual(response.body, utf8(method))
-        response = self.fetch('/all_methods', method='HEAD')
-        self.assertEqual(response.body, b'')
-        response = self.fetch('/all_methods', method='OTHER',
-                              allow_nonstandard_methods=True)
-        self.assertEqual(response.body, b'OTHER')
-
-    def test_body_sanity_checks(self):
-        # These methods require a body.
-        for method in ('POST', 'PUT', 'PATCH'):
-            with self.assertRaises(ValueError) as context:
-                resp = self.fetch('/all_methods', method=method)
-                resp.rethrow()
-            self.assertIn('must not be None', str(context.exception))
-
-            resp = self.fetch('/all_methods', method=method,
-                              allow_nonstandard_methods=True)
-            self.assertEqual(resp.code, 200)
-
-        # These methods don't allow a body.
-        for method in ('GET', 'DELETE', 'OPTIONS'):
-            with self.assertRaises(ValueError) as context:
-                resp = self.fetch('/all_methods', method=method, body=b'asdf')
-                resp.rethrow()
-            self.assertIn('must be None', str(context.exception))
-
-            # In most cases this can be overridden, but curl_httpclient
-            # does not allow body with a GET at all.
-            if method != 'GET':
-                resp = self.fetch('/all_methods', method=method, body=b'asdf',
-                                  allow_nonstandard_methods=True)
-                resp.rethrow()
-                self.assertEqual(resp.code, 200)
-
-    # This test causes odd failures with the combination of
-    # curl_httpclient (at least with the version of libcurl available
-    # on ubuntu 12.04), TwistedIOLoop, and epoll.  For POST (but not PUT),
-    # curl decides the response came back too soon and closes the connection
-    # to start again.  It does this *before* telling the socket callback to
-    # unregister the FD.  Some IOLoop implementations have special kernel
-    # integration to discover this immediately.  Tornado's IOLoops
-    # ignore errors on remove_handler to accommodate this behavior, but
-    # Twisted's reactor does not.  The removeReader call fails and so
-    # do all future removeAll calls (which our tests do at cleanup).
-    #
-    # def test_post_307(self):
-    #    response = self.fetch("/redirect?status=307&url=/post",
-    #                          method="POST", body=b"arg1=foo&arg2=bar")
-    #    self.assertEqual(response.body, b"Post arg1: foo, arg2: bar")
-
-    def test_put_307(self):
-        response = self.fetch("/redirect?status=307&url=/put",
-                              method="PUT", body=b"hello")
-        response.rethrow()
-        self.assertEqual(response.body, b"Put body: hello")
-
-    def test_non_ascii_header(self):
-        # Non-ascii headers are sent as latin1.
-        response = self.fetch("/set_header?k=foo&v=%E9")
-        response.rethrow()
-        self.assertEqual(response.headers["Foo"], native_str(u"\u00e9"))
-
-
-class RequestProxyTest(unittest.TestCase):
-    def test_request_set(self):
-        proxy = _RequestProxy(HTTPRequest('http://example.com/',
-                                          user_agent='foo'),
-                              dict())
-        self.assertEqual(proxy.user_agent, 'foo')
-
-    def test_default_set(self):
-        proxy = _RequestProxy(HTTPRequest('http://example.com/'),
-                              dict(network_interface='foo'))
-        self.assertEqual(proxy.network_interface, 'foo')
-
-    def test_both_set(self):
-        proxy = _RequestProxy(HTTPRequest('http://example.com/',
-                                          proxy_host='foo'),
-                              dict(proxy_host='bar'))
-        self.assertEqual(proxy.proxy_host, 'foo')
-
-    def test_neither_set(self):
-        proxy = _RequestProxy(HTTPRequest('http://example.com/'),
-                              dict())
-        self.assertIs(proxy.auth_username, None)
-
-    def test_bad_attribute(self):
-        proxy = _RequestProxy(HTTPRequest('http://example.com/'),
-                              dict())
-        with self.assertRaises(AttributeError):
-            proxy.foo
-
-    def test_defaults_none(self):
-        proxy = _RequestProxy(HTTPRequest('http://example.com/'), None)
-        self.assertIs(proxy.auth_username, None)
-
-
-class HTTPResponseTestCase(unittest.TestCase):
-    def test_str(self):
-        response = HTTPResponse(HTTPRequest('http://example.com'),
-                                200, headers={}, buffer=BytesIO())
-        s = str(response)
-        self.assertTrue(s.startswith('HTTPResponse('))
-        self.assertIn('code=200', s)
-
-
-class SyncHTTPClientTest(unittest.TestCase):
-    def setUp(self):
-        if IOLoop.configured_class().__name__ in ('TwistedIOLoop',
-                                                  'AsyncIOMainLoop'):
-            # TwistedIOLoop only supports the global reactor, so we can't have
-            # separate IOLoops for client and server threads.
-            # AsyncIOMainLoop doesn't work with the default policy
-            # (although it could with some tweaks to this test and a
-            # policy that created loops for non-main threads).
-            raise unittest.SkipTest(
-                'Sync HTTPClient not compatible with TwistedIOLoop or '
-                'AsyncIOMainLoop')
-        self.server_ioloop = IOLoop()
-
-        sock, self.port = bind_unused_port()
-        app = Application([('/', HelloWorldHandler)])
-        self.server = HTTPServer(app, io_loop=self.server_ioloop)
-        self.server.add_socket(sock)
-
-        self.server_thread = threading.Thread(target=self.server_ioloop.start)
-        self.server_thread.start()
-
-        self.http_client = HTTPClient()
-
-    def tearDown(self):
-        def stop_server():
-            self.server.stop()
-            # Delay the shutdown of the IOLoop by one iteration because
-            # the server may still have some cleanup work left when
-            # the client finishes with the response (this is noticable
-            # with http/2, which leaves a Future with an unexamined
-            # StreamClosedError on the loop).
-            self.server_ioloop.add_callback(self.server_ioloop.stop)
-        self.server_ioloop.add_callback(stop_server)
-        self.server_thread.join()
-        self.http_client.close()
-        self.server_ioloop.close(all_fds=True)
-
-    def get_url(self, path):
-        return 'http://127.0.0.1:%d%s' % (self.port, path)
-
-    def test_sync_client(self):
-        response = self.http_client.fetch(self.get_url('/'))
-        self.assertEqual(b'Hello world!', response.body)
-
-    def test_sync_client_error(self):
-        # Synchronous HTTPClient raises errors directly; no need for
-        # response.rethrow()
-        with self.assertRaises(HTTPError) as assertion:
-            self.http_client.fetch(self.get_url('/notfound'))
-        self.assertEqual(assertion.exception.code, 404)
-
-
-class HTTPRequestTestCase(unittest.TestCase):
-    def test_headers(self):
-        request = HTTPRequest('http://example.com', headers={'foo': 'bar'})
-        self.assertEqual(request.headers, {'foo': 'bar'})
-
-    def test_headers_setter(self):
-        request = HTTPRequest('http://example.com')
-        request.headers = {'bar': 'baz'}
-        self.assertEqual(request.headers, {'bar': 'baz'})
-
-    def test_null_headers_setter(self):
-        request = HTTPRequest('http://example.com')
-        request.headers = None
-        self.assertEqual(request.headers, {})
-
-    def test_body(self):
-        request = HTTPRequest('http://example.com', body='foo')
-        self.assertEqual(request.body, utf8('foo'))
-
-    def test_body_setter(self):
-        request = HTTPRequest('http://example.com')
-        request.body = 'foo'
-        self.assertEqual(request.body, utf8('foo'))
-
-    def test_if_modified_since(self):
-        http_date = datetime.datetime.utcnow()
-        request = HTTPRequest('http://example.com', if_modified_since=http_date)
-        self.assertEqual(request.headers,
-                         {'If-Modified-Since': format_timestamp(http_date)})
-
-
-class HTTPErrorTestCase(unittest.TestCase):
-    def test_copy(self):
-        e = HTTPError(403)
-        e2 = copy.copy(e)
-        self.assertIsNot(e, e2)
-        self.assertEqual(e.code, e2.code)
-
-    def test_plain_error(self):
-        e = HTTPError(403)
-        self.assertEqual(str(e), "HTTP 403: Forbidden")
-        self.assertEqual(repr(e), "HTTP 403: Forbidden")
-
-    def test_error_with_response(self):
-        resp = HTTPResponse(HTTPRequest('http://example.com/'), 403)
-        with self.assertRaises(HTTPError) as cm:
-            resp.rethrow()
-        e = cm.exception
-        self.assertEqual(str(e), "HTTP 403: Forbidden")
-        self.assertEqual(repr(e), "HTTP 403: Forbidden")
diff --git a/salt/ext/tornado/test/httpserver_test.py b/salt/ext/tornado/test/httpserver_test.py
deleted file mode 100644
index 7ef5dfc78f4..00000000000
--- a/salt/ext/tornado/test/httpserver_test.py
+++ /dev/null
@@ -1,1135 +0,0 @@
-#!/usr/bin/env python
-# pylint: skip-file
-
-
-from __future__ import absolute_import, division, print_function
-from salt.ext.tornado import netutil
-from salt.ext.tornado.escape import json_decode, json_encode, utf8, _unicode, recursive_unicode, native_str
-from salt.ext.tornado import gen
-from salt.ext.tornado.http1connection import HTTP1Connection
-from salt.ext.tornado.httpserver import HTTPServer
-from salt.ext.tornado.httputil import HTTPHeaders, HTTPMessageDelegate, HTTPServerConnectionDelegate, ResponseStartLine
-from salt.ext.tornado.iostream import IOStream
-from salt.ext.tornado.log import gen_log
-from salt.ext.tornado.netutil import ssl_options_to_context
-from salt.ext.tornado.simple_httpclient import SimpleAsyncHTTPClient
-from salt.ext.tornado.testing import AsyncHTTPTestCase, AsyncHTTPSTestCase, AsyncTestCase, ExpectLog, gen_test
-from salt.ext.tornado.test.util import unittest, skipOnTravis
-from salt.ext.tornado.web import Application, RequestHandler, asynchronous, stream_request_body
-from contextlib import closing
-import datetime
-import gzip
-import os
-import shutil
-import socket
-import ssl
-import sys
-import tempfile
-from io import BytesIO
-
-
-def read_stream_body(stream, callback):
-    """Reads an HTTP response from `stream` and runs callback with its
-    headers and body."""
-    chunks = []
-
-    class Delegate(HTTPMessageDelegate):
-        def headers_received(self, start_line, headers):
-            self.headers = headers
-
-        def data_received(self, chunk):
-            chunks.append(chunk)
-
-        def finish(self):
-            callback((self.headers, b''.join(chunks)))
-    conn = HTTP1Connection(stream, True)
-    conn.read_response(Delegate())
-
-
-class HandlerBaseTestCase(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application([('/', self.__class__.Handler)])
-
-    def fetch_json(self, *args, **kwargs):
-        response = self.fetch(*args, **kwargs)
-        response.rethrow()
-        return json_decode(response.body)
-
-
-class HelloWorldRequestHandler(RequestHandler):
-    def initialize(self, protocol="http"):
-        self.expected_protocol = protocol
-
-    def get(self):
-        if self.request.protocol != self.expected_protocol:
-            raise Exception("unexpected protocol")
-        self.finish("Hello world")
-
-    def post(self):
-        self.finish("Got %d bytes in POST" % len(self.request.body))
-
-
-# In pre-1.0 versions of openssl, SSLv23 clients always send SSLv2
-# ClientHello messages, which are rejected by SSLv3 and TLSv1
-# servers.  Note that while the OPENSSL_VERSION_INFO was formally
-# introduced in python3.2, it was present but undocumented in
-# python 2.7
-skipIfOldSSL = unittest.skipIf(
-    getattr(ssl, 'OPENSSL_VERSION_INFO', (0, 0)) < (1, 0),
-    "old version of ssl module and/or openssl")
-
-
-class BaseSSLTest(AsyncHTTPSTestCase):
-    def get_app(self):
-        return Application([('/', HelloWorldRequestHandler,
-                             dict(protocol="https"))])
-
-
-class SSLTestMixin(object):
-    def get_ssl_options(self):
-        return dict(ssl_version=self.get_ssl_version(),  # type: ignore
-                    **AsyncHTTPSTestCase.get_ssl_options())
-
-    def get_ssl_version(self):
-        raise NotImplementedError()
-
-    def test_ssl(self):
-        response = self.fetch('/')
-        self.assertEqual(response.body, b"Hello world")
-
-    def test_large_post(self):
-        response = self.fetch('/',
-                              method='POST',
-                              body='A' * 5000)
-        self.assertEqual(response.body, b"Got 5000 bytes in POST")
-
-    def test_non_ssl_request(self):
-        # Make sure the server closes the connection when it gets a non-ssl
-        # connection, rather than waiting for a timeout or otherwise
-        # misbehaving.
-        with ExpectLog(gen_log, '(SSL Error|uncaught exception)'):
-            with ExpectLog(gen_log, 'Uncaught exception', required=False):
-                self.http_client.fetch(
-                    self.get_url("/").replace('https:', 'http:'),
-                    self.stop,
-                    request_timeout=3600,
-                    connect_timeout=3600)
-                response = self.wait()
-        self.assertEqual(response.code, 599)
-
-    def test_error_logging(self):
-        # No stack traces are logged for SSL errors.
-        with ExpectLog(gen_log, 'SSL Error') as expect_log:
-            self.http_client.fetch(
-                self.get_url("/").replace("https:", "http:"),
-                self.stop)
-            response = self.wait()
-            self.assertEqual(response.code, 599)
-        self.assertFalse(expect_log.logged_stack)
-
-# Python's SSL implementation differs significantly between versions.
-# For example, SSLv3 and TLSv1 throw an exception if you try to read
-# from the socket before the handshake is complete, but the default
-# of SSLv23 allows it.
-
-
-class SSLv23Test(BaseSSLTest, SSLTestMixin):
-    def get_ssl_version(self):
-        return ssl.PROTOCOL_SSLv23
-
-
-@skipIfOldSSL
-class SSLv3Test(BaseSSLTest, SSLTestMixin):
-    def get_ssl_version(self):
-        return ssl.PROTOCOL_SSLv3
-
-
-@skipIfOldSSL
-class TLSv1Test(BaseSSLTest, SSLTestMixin):
-    def get_ssl_version(self):
-        return ssl.PROTOCOL_TLSv1
-
-
-@unittest.skipIf(not hasattr(ssl, 'SSLContext'), 'ssl.SSLContext not present')
-class SSLContextTest(BaseSSLTest, SSLTestMixin):
-    def get_ssl_options(self):
-        context = ssl_options_to_context(
-            AsyncHTTPSTestCase.get_ssl_options(self))
-        assert isinstance(context, ssl.SSLContext)
-        return context
-
-
-class BadSSLOptionsTest(unittest.TestCase):
-    def test_missing_arguments(self):
-        application = Application()
-        self.assertRaises(KeyError, HTTPServer, application, ssl_options={
-            "keyfile": "/__missing__.crt",
-        })
-
-    def test_missing_key(self):
-        """A missing SSL key should cause an immediate exception."""
-
-        application = Application()
-        module_dir = os.path.dirname(__file__)
-        existing_certificate = os.path.join(module_dir, 'test.crt')
-        existing_key = os.path.join(module_dir, 'test.key')
-
-        self.assertRaises((ValueError, IOError),
-                          HTTPServer, application, ssl_options={
-                              "certfile": "/__mising__.crt",
-        })
-        self.assertRaises((ValueError, IOError),
-                          HTTPServer, application, ssl_options={
-                              "certfile": existing_certificate,
-                              "keyfile": "/__missing__.key"
-        })
-
-        # This actually works because both files exist
-        HTTPServer(application, ssl_options={
-                   "certfile": existing_certificate,
-                   "keyfile": existing_key,
-                   })
-
-
-class MultipartTestHandler(RequestHandler):
-    def post(self):
-        self.finish({"header": self.request.headers["X-Header-Encoding-Test"],
-                     "argument": self.get_argument("argument"),
-                     "filename": self.request.files["files"][0].filename,
-                     "filebody": _unicode(self.request.files["files"][0]["body"]),
-                     })
-
-
-# This test is also called from wsgi_test
-class HTTPConnectionTest(AsyncHTTPTestCase):
-    def get_handlers(self):
-        return [("/multipart", MultipartTestHandler),
-                ("/hello", HelloWorldRequestHandler)]
-
-    def get_app(self):
-        return Application(self.get_handlers())
-
-    def raw_fetch(self, headers, body, newline=b"\r\n"):
-        with closing(IOStream(socket.socket())) as stream:
-            stream.connect(('127.0.0.1', self.get_http_port()), self.stop)
-            self.wait()
-            stream.write(
-                newline.join(headers +
-                             [utf8("Content-Length: %d" % len(body))]) +
-                newline + newline + body)
-            read_stream_body(stream, self.stop)
-            headers, body = self.wait()
-            return body
-
-    def test_multipart_form(self):
-        # Encodings here are tricky:  Headers are latin1, bodies can be
-        # anything (we use utf8 by default).
-        response = self.raw_fetch([
-            b"POST /multipart HTTP/1.0",
-            b"Content-Type: multipart/form-data; boundary=1234567890",
-            b"X-Header-encoding-test: \xe9",
-        ],
-            b"\r\n".join([
-                b"Content-Disposition: form-data; name=argument",
-                b"",
-                u"\u00e1".encode("utf-8"),
-                b"--1234567890",
-                u'Content-Disposition: form-data; name="files"; filename="\u00f3"'.encode("utf8"),
-                b"",
-                u"\u00fa".encode("utf-8"),
-                b"--1234567890--",
-                b"",
-            ]))
-        data = json_decode(response)
-        self.assertEqual(u"\u00e9", data["header"])
-        self.assertEqual(u"\u00e1", data["argument"])
-        self.assertEqual(u"\u00f3", data["filename"])
-        self.assertEqual(u"\u00fa", data["filebody"])
-
-    def test_newlines(self):
-        # We support both CRLF and bare LF as line separators.
-        for newline in (b"\r\n", b"\n"):
-            response = self.raw_fetch([b"GET /hello HTTP/1.0"], b"",
-                                      newline=newline)
-            self.assertEqual(response, b'Hello world')
-
-    def test_100_continue(self):
-        # Run through a 100-continue interaction by hand:
-        # When given Expect: 100-continue, we get a 100 response after the
-        # headers, and then the real response after the body.
-        stream = IOStream(socket.socket(), io_loop=self.io_loop)
-        stream.connect(("127.0.0.1", self.get_http_port()), callback=self.stop)
-        self.wait()
-        stream.write(b"\r\n".join([b"POST /hello HTTP/1.1",
-                                   b"Content-Length: 1024",
-                                   b"Expect: 100-continue",
-                                   b"Connection: close",
-                                   b"\r\n"]), callback=self.stop)
-        self.wait()
-        stream.read_until(b"\r\n\r\n", self.stop)
-        data = self.wait()
-        self.assertTrue(data.startswith(b"HTTP/1.1 100 "), data)
-        stream.write(b"a" * 1024)
-        stream.read_until(b"\r\n", self.stop)
-        first_line = self.wait()
-        self.assertTrue(first_line.startswith(b"HTTP/1.1 200"), first_line)
-        stream.read_until(b"\r\n\r\n", self.stop)
-        header_data = self.wait()
-        headers = HTTPHeaders.parse(native_str(header_data.decode('latin1')))
-        stream.read_bytes(int(headers["Content-Length"]), self.stop)
-        body = self.wait()
-        self.assertEqual(body, b"Got 1024 bytes in POST")
-        stream.close()
-
-
-class EchoHandler(RequestHandler):
-    def get(self):
-        self.write(recursive_unicode(self.request.arguments))
-
-    def post(self):
-        self.write(recursive_unicode(self.request.arguments))
-
-
-class TypeCheckHandler(RequestHandler):
-    def prepare(self):
-        self.errors = {}
-        fields = [
-            ('method', str),
-            ('uri', str),
-            ('version', str),
-            ('remote_ip', str),
-            ('protocol', str),
-            ('host', str),
-            ('path', str),
-            ('query', str),
-        ]
-        for field, expected_type in fields:
-            self.check_type(field, getattr(self.request, field), expected_type)
-
-        self.check_type('header_key', list(self.request.headers.keys())[0], str)
-        self.check_type('header_value', list(self.request.headers.values())[0], str)
-
-        self.check_type('cookie_key', list(self.request.cookies.keys())[0], str)
-        self.check_type('cookie_value', list(self.request.cookies.values())[0].value, str)
-        # secure cookies
-
-        self.check_type('arg_key', list(self.request.arguments.keys())[0], str)
-        self.check_type('arg_value', list(self.request.arguments.values())[0][0], bytes)
-
-    def post(self):
-        self.check_type('body', self.request.body, bytes)
-        self.write(self.errors)
-
-    def get(self):
-        self.write(self.errors)
-
-    def check_type(self, name, obj, expected_type):
-        actual_type = type(obj)
-        if expected_type != actual_type:
-            self.errors[name] = "expected %s, got %s" % (expected_type,
-                                                         actual_type)
-
-
-class HTTPServerTest(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application([("/echo", EchoHandler),
-                            ("/typecheck", TypeCheckHandler),
-                            ("//doubleslash", EchoHandler),
-                            ])
-
-    def test_query_string_encoding(self):
-        response = self.fetch("/echo?foo=%C3%A9")
-        data = json_decode(response.body)
-        self.assertEqual(data, {u"foo": [u"\u00e9"]})
-
-    def test_empty_query_string(self):
-        response = self.fetch("/echo?foo=&foo=")
-        data = json_decode(response.body)
-        self.assertEqual(data, {u"foo": [u"", u""]})
-
-    def test_empty_post_parameters(self):
-        response = self.fetch("/echo", method="POST", body="foo=&bar=")
-        data = json_decode(response.body)
-        self.assertEqual(data, {u"foo": [u""], u"bar": [u""]})
-
-    def test_types(self):
-        headers = {"Cookie": "foo=bar"}
-        response = self.fetch("/typecheck?foo=bar", headers=headers)
-        data = json_decode(response.body)
-        self.assertEqual(data, {})
-
-        response = self.fetch("/typecheck", method="POST", body="foo=bar", headers=headers)
-        data = json_decode(response.body)
-        self.assertEqual(data, {})
-
-    def test_double_slash(self):
-        # urlparse.urlsplit (which tornado.httpserver used to use
-        # incorrectly) would parse paths beginning with "//" as
-        # protocol-relative urls.
-        response = self.fetch("//doubleslash")
-        self.assertEqual(200, response.code)
-        self.assertEqual(json_decode(response.body), {})
-
-    def test_malformed_body(self):
-        # parse_qs is pretty forgiving, but it will fail on python 3
-        # if the data is not utf8.  On python 2 parse_qs will work,
-        # but then the recursive_unicode call in EchoHandler will
-        # fail.
-        if str is bytes:
-            return
-        with ExpectLog(gen_log, 'Invalid x-www-form-urlencoded body'):
-            response = self.fetch(
-                '/echo', method="POST",
-                headers={'Content-Type': 'application/x-www-form-urlencoded'},
-                body=b'\xe9')
-        self.assertEqual(200, response.code)
-        self.assertEqual(b'{}', response.body)
-
-
-class HTTPServerRawTest(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application([
-            ('/echo', EchoHandler),
-        ])
-
-    def setUp(self):
-        super(HTTPServerRawTest, self).setUp()
-        self.stream = IOStream(socket.socket())
-        self.stream.connect(('127.0.0.1', self.get_http_port()), self.stop)
-        self.wait()
-
-    def tearDown(self):
-        self.stream.close()
-        super(HTTPServerRawTest, self).tearDown()
-
-    def test_empty_request(self):
-        self.stream.close()
-        self.io_loop.add_timeout(datetime.timedelta(seconds=0.001), self.stop)
-        self.wait()
-
-    def test_malformed_first_line(self):
-        with ExpectLog(gen_log, '.*Malformed HTTP request line'):
-            self.stream.write(b'asdf\r\n\r\n')
-            # TODO: need an async version of ExpectLog so we don't need
-            # hard-coded timeouts here.
-            self.io_loop.add_timeout(datetime.timedelta(seconds=0.05),
-                                     self.stop)
-            self.wait()
-
-    def test_malformed_headers(self):
-        with ExpectLog(gen_log, '.*Malformed HTTP headers'):
-            self.stream.write(b'GET / HTTP/1.0\r\nasdf\r\n\r\n')
-            self.io_loop.add_timeout(datetime.timedelta(seconds=0.05),
-                                     self.stop)
-            self.wait()
-
-    def test_chunked_request_body(self):
-        # Chunked requests are not widely supported and we don't have a way
-        # to generate them in AsyncHTTPClient, but HTTPServer will read them.
-        self.stream.write(b"""\
-POST /echo HTTP/1.1
-Transfer-Encoding: chunked
-Content-Type: application/x-www-form-urlencoded
-
-4
-foo=
-3
-bar
-0
-
-""".replace(b"\n", b"\r\n"))
-        read_stream_body(self.stream, self.stop)
-        headers, response = self.wait()
-        self.assertEqual(json_decode(response), {u'foo': [u'bar']})
-
-    def test_chunked_request_uppercase(self):
-        # As per RFC 2616 section 3.6, "Transfer-Encoding" header's value is
-        # case-insensitive.
-        self.stream.write(b"""\
-POST /echo HTTP/1.1
-Transfer-Encoding: Chunked
-Content-Type: application/x-www-form-urlencoded
-
-4
-foo=
-3
-bar
-0
-
-""".replace(b"\n", b"\r\n"))
-        read_stream_body(self.stream, self.stop)
-        headers, response = self.wait()
-        self.assertEqual(json_decode(response), {u'foo': [u'bar']})
-
-    def test_invalid_content_length(self):
-        with ExpectLog(gen_log, '.*Only integer Content-Length is allowed'):
-            self.stream.write(b"""\
-POST /echo HTTP/1.1
-Content-Length: foo
-
-bar
-
-""".replace(b"\n", b"\r\n"))
-            self.stream.read_until_close(self.stop)
-            self.wait()
-
-
-class XHeaderTest(HandlerBaseTestCase):
-    class Handler(RequestHandler):
-        def get(self):
-            self.write(dict(remote_ip=self.request.remote_ip,
-                            remote_protocol=self.request.protocol))
-
-    def get_httpserver_options(self):
-        return dict(xheaders=True, trusted_downstream=['5.5.5.5'])
-
-    def test_ip_headers(self):
-        self.assertEqual(self.fetch_json("/")["remote_ip"], "127.0.0.1")
-
-        valid_ipv4 = {"X-Real-IP": "4.4.4.4"}
-        self.assertEqual(
-            self.fetch_json("/", headers=valid_ipv4)["remote_ip"],
-            "4.4.4.4")
-
-        valid_ipv4_list = {"X-Forwarded-For": "127.0.0.1, 4.4.4.4"}
-        self.assertEqual(
-            self.fetch_json("/", headers=valid_ipv4_list)["remote_ip"],
-            "4.4.4.4")
-
-        valid_ipv6 = {"X-Real-IP": "2620:0:1cfe:face:b00c::3"}
-        self.assertEqual(
-            self.fetch_json("/", headers=valid_ipv6)["remote_ip"],
-            "2620:0:1cfe:face:b00c::3")
-
-        valid_ipv6_list = {"X-Forwarded-For": "::1, 2620:0:1cfe:face:b00c::3"}
-        self.assertEqual(
-            self.fetch_json("/", headers=valid_ipv6_list)["remote_ip"],
-            "2620:0:1cfe:face:b00c::3")
-
-        invalid_chars = {"X-Real-IP": "4.4.4.4
-
-'
-                       for p in paths)
-
-    def render_embed_js(self, js_embed):
-        """Default method used to render the final embedded js for the
-        rendered webpage.
-
-        Override this method in a sub-classed controller to change the output.
-        """
-        return b''
-
-    def render_linked_css(self, css_files):
-        """Default method used to render the final css links for the
-        rendered webpage.
-
-        Override this method in a sub-classed controller to change the output.
-        """
-        paths = []
-        unique_paths = set()
-
-        for path in css_files:
-            if not is_absolute(path):
-                path = self.static_url(path)
-            if path not in unique_paths:
-                paths.append(path)
-                unique_paths.add(path)
-
-        return ''.join(''
-                       for p in paths)
-
-    def render_embed_css(self, css_embed):
-        """Default method used to render the final embedded css for the
-        rendered webpage.
-
-        Override this method in a sub-classed controller to change the output.
-        """
-        return b''
-
-    def render_string(self, template_name, **kwargs):
-        """Generate the given template with the given arguments.
-
-        We return the generated byte string (in utf8). To generate and
-        write a template as a response, use render() above.
-        """
-        # If no template_path is specified, use the path of the calling file
-        template_path = self.get_template_path()
-        if not template_path:
-            frame = sys._getframe(0)
-            web_file = frame.f_code.co_filename
-            while frame.f_code.co_filename == web_file:
-                frame = frame.f_back
-            template_path = os.path.dirname(frame.f_code.co_filename)
-        with RequestHandler._template_loader_lock:
-            if template_path not in RequestHandler._template_loaders:
-                loader = self.create_template_loader(template_path)
-                RequestHandler._template_loaders[template_path] = loader
-            else:
-                loader = RequestHandler._template_loaders[template_path]
-        t = loader.load(template_name)
-        namespace = self.get_template_namespace()
-        namespace.update(kwargs)
-        return t.generate(**namespace)
-
-    def get_template_namespace(self):
-        """Returns a dictionary to be used as the default template namespace.
-
-        May be overridden by subclasses to add or modify values.
-
-        The results of this method will be combined with additional
-        defaults in the `tornado.template` module and keyword arguments
-        to `render` or `render_string`.
-        """
-        namespace = dict(
-            handler=self,
-            request=self.request,
-            current_user=self.current_user,
-            locale=self.locale,
-            _=self.locale.translate,
-            pgettext=self.locale.pgettext,
-            static_url=self.static_url,
-            xsrf_form_html=self.xsrf_form_html,
-            reverse_url=self.reverse_url
-        )
-        namespace.update(self.ui)
-        return namespace
-
-    def create_template_loader(self, template_path):
-        """Returns a new template loader for the given path.
-
-        May be overridden by subclasses.  By default returns a
-        directory-based loader on the given path, using the
-        ``autoescape`` and ``template_whitespace`` application
-        settings.  If a ``template_loader`` application setting is
-        supplied, uses that instead.
-        """
-        settings = self.application.settings
-        if "template_loader" in settings:
-            return settings["template_loader"]
-        kwargs = {}
-        if "autoescape" in settings:
-            # autoescape=None means "no escaping", so we have to be sure
-            # to only pass this kwarg if the user asked for it.
-            kwargs["autoescape"] = settings["autoescape"]
-        if "template_whitespace" in settings:
-            kwargs["whitespace"] = settings["template_whitespace"]
-        return template.Loader(template_path, **kwargs)
-
-    def flush(self, include_footers=False, callback=None):
-        """Flushes the current output buffer to the network.
-
-        The ``callback`` argument, if given, can be used for flow control:
-        it will be run when all flushed data has been written to the socket.
-        Note that only one flush callback can be outstanding at a time;
-        if another flush occurs before the previous flush's callback
-        has been run, the previous callback will be discarded.
-
-        .. versionchanged:: 4.0
-           Now returns a `.Future` if no callback is given.
-        """
-        chunk = b"".join(self._write_buffer)
-        self._write_buffer = []
-        if not self._headers_written:
-            self._headers_written = True
-            for transform in self._transforms:
-                self._status_code, self._headers, chunk = \
-                    transform.transform_first_chunk(
-                        self._status_code, self._headers,
-                        chunk, include_footers)
-            # Ignore the chunk and only write the headers for HEAD requests
-            if self.request.method == "HEAD":
-                chunk = None
-
-            # Finalize the cookie headers (which have been stored in a side
-            # object so an outgoing cookie could be overwritten before it
-            # is sent).
-            if hasattr(self, "_new_cookie"):
-                for cookie in self._new_cookie.values():
-                    self.add_header("Set-Cookie", cookie.OutputString(None))
-
-            start_line = httputil.ResponseStartLine('',
-                                                    self._status_code,
-                                                    self._reason)
-            return self.request.connection.write_headers(
-                start_line, self._headers, chunk, callback=callback)
-        else:
-            for transform in self._transforms:
-                chunk = transform.transform_chunk(chunk, include_footers)
-            # Ignore the chunk and only write the headers for HEAD requests
-            if self.request.method != "HEAD":
-                return self.request.connection.write(chunk, callback=callback)
-            else:
-                future = Future()
-                future.set_result(None)
-                return future
-
-    def finish(self, chunk=None):
-        """Finishes this response, ending the HTTP request."""
-        if self._finished:
-            raise RuntimeError("finish() called twice")
-
-        if chunk is not None:
-            self.write(chunk)
-
-        # Automatically support ETags and add the Content-Length header if
-        # we have not flushed any content yet.
-        if not self._headers_written:
-            if (self._status_code == 200 and
-                self.request.method in ("GET", "HEAD") and
-                    "Etag" not in self._headers):
-                self.set_etag_header()
-                if self.check_etag_header():
-                    self._write_buffer = []
-                    self.set_status(304)
-            if (self._status_code in (204, 304) or
-                (self._status_code >= 100 and self._status_code < 200)):
-                assert not self._write_buffer, "Cannot send body with %s" % self._status_code
-                self._clear_headers_for_304()
-            elif "Content-Length" not in self._headers:
-                content_length = sum(len(part) for part in self._write_buffer)
-                self.set_header("Content-Length", content_length)
-
-        if hasattr(self.request, "connection"):
-            # Now that the request is finished, clear the callback we
-            # set on the HTTPConnection (which would otherwise prevent the
-            # garbage collection of the RequestHandler when there
-            # are keepalive connections)
-            self.request.connection.set_close_callback(None)
-
-        self.flush(include_footers=True)
-        self.request.finish()
-        self._log()
-        self._finished = True
-        self.on_finish()
-        self._break_cycles()
-
-    def _break_cycles(self):
-        # Break up a reference cycle between this handler and the
-        # _ui_module closures to allow for faster GC on CPython.
-        self.ui = None
-
-    def send_error(self, status_code=500, **kwargs):
-        """Sends the given HTTP error code to the browser.
-
-        If `flush()` has already been called, it is not possible to send
-        an error, so this method will simply terminate the response.
-        If output has been written but not yet flushed, it will be discarded
-        and replaced with the error page.
-
-        Override `write_error()` to customize the error page that is returned.
-        Additional keyword arguments are passed through to `write_error`.
-        """
-        if self._headers_written:
-            gen_log.error("Cannot send error response after headers written")
-            if not self._finished:
-                # If we get an error between writing headers and finishing,
-                # we are unlikely to be able to finish due to a
-                # Content-Length mismatch. Try anyway to release the
-                # socket.
-                try:
-                    self.finish()
-                except Exception:
-                    gen_log.error("Failed to flush partial response",
-                                  exc_info=True)
-            return
-        self.clear()
-
-        reason = kwargs.get('reason')
-        if 'exc_info' in kwargs:
-            exception = kwargs['exc_info'][1]
-            if isinstance(exception, HTTPError) and exception.reason:
-                reason = exception.reason
-        self.set_status(status_code, reason=reason)
-        try:
-            self.write_error(status_code, **kwargs)
-        except Exception:
-            app_log.error("Uncaught exception in write_error", exc_info=True)
-        if not self._finished:
-            self.finish()
-
-    def write_error(self, status_code, **kwargs):
-        """Override to implement custom error pages.
-
-        ``write_error`` may call `write`, `render`, `set_header`, etc
-        to produce output as usual.
-
-        If this error was caused by an uncaught exception (including
-        HTTPError), an ``exc_info`` triple will be available as
-        ``kwargs["exc_info"]``.  Note that this exception may not be
-        the "current" exception for purposes of methods like
-        ``sys.exc_info()`` or ``traceback.format_exc``.
-        """
-        if self.settings.get("serve_traceback") and "exc_info" in kwargs:
-            # in debug mode, try to send a traceback
-            self.set_header('Content-Type', 'text/plain')
-            for line in traceback.format_exception(*kwargs["exc_info"]):
-                self.write(line)
-            self.finish()
-        else:
-            self.finish("%(code)d: %(message)s"
-                        "%(code)d: %(message)s" % {
-                            "code": status_code,
-                            "message": self._reason,
-                        })
-
-    @property
-    def locale(self):
-        """The locale for the current session.
-
-        Determined by either `get_user_locale`, which you can override to
-        set the locale based on, e.g., a user preference stored in a
-        database, or `get_browser_locale`, which uses the ``Accept-Language``
-        header.
-
-        .. versionchanged: 4.1
-           Added a property setter.
-        """
-        if not hasattr(self, "_locale"):
-            self._locale = self.get_user_locale()
-            if not self._locale:
-                self._locale = self.get_browser_locale()
-                assert self._locale
-        return self._locale
-
-    @locale.setter
-    def locale(self, value):
-        self._locale = value
-
-    def get_user_locale(self):
-        """Override to determine the locale from the authenticated user.
-
-        If None is returned, we fall back to `get_browser_locale()`.
-
-        This method should return a `tornado.locale.Locale` object,
-        most likely obtained via a call like ``tornado.locale.get("en")``
-        """
-        return None
-
-    def get_browser_locale(self, default="en_US"):
-        """Determines the user's locale from ``Accept-Language`` header.
-
-        See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
-        """
-        if "Accept-Language" in self.request.headers:
-            languages = self.request.headers["Accept-Language"].split(",")
-            locales = []
-            for language in languages:
-                parts = language.strip().split(";")
-                if len(parts) > 1 and parts[1].startswith("q="):
-                    try:
-                        score = float(parts[1][2:])
-                    except (ValueError, TypeError):
-                        score = 0.0
-                else:
-                    score = 1.0
-                locales.append((parts[0], score))
-            if locales:
-                locales.sort(key=lambda pair: pair[1], reverse=True)
-                codes = [l[0] for l in locales]
-                return locale.get(*codes)
-        return locale.get(default)
-
-    @property
-    def current_user(self):
-        """The authenticated user for this request.
-
-        This is set in one of two ways:
-
-        * A subclass may override `get_current_user()`, which will be called
-          automatically the first time ``self.current_user`` is accessed.
-          `get_current_user()` will only be called once per request,
-          and is cached for future access::
-
-              def get_current_user(self):
-                  user_cookie = self.get_secure_cookie("user")
-                  if user_cookie:
-                      return json.loads(user_cookie)
-                  return None
-
-        * It may be set as a normal variable, typically from an overridden
-          `prepare()`::
-
-              @gen.coroutine
-              def prepare(self):
-                  user_id_cookie = self.get_secure_cookie("user_id")
-                  if user_id_cookie:
-                      self.current_user = yield load_user(user_id_cookie)
-
-        Note that `prepare()` may be a coroutine while `get_current_user()`
-        may not, so the latter form is necessary if loading the user requires
-        asynchronous operations.
-
-        The user object may be any type of the application's choosing.
-        """
-        if not hasattr(self, "_current_user"):
-            self._current_user = self.get_current_user()
-        return self._current_user
-
-    @current_user.setter
-    def current_user(self, value):
-        self._current_user = value
-
-    def get_current_user(self):
-        """Override to determine the current user from, e.g., a cookie.
-
-        This method may not be a coroutine.
-        """
-        return None
-
-    def get_login_url(self):
-        """Override to customize the login URL based on the request.
-
-        By default, we use the ``login_url`` application setting.
-        """
-        self.require_setting("login_url", "@tornado.web.authenticated")
-        return self.application.settings["login_url"]
-
-    def get_template_path(self):
-        """Override to customize template path for each handler.
-
-        By default, we use the ``template_path`` application setting.
-        Return None to load templates relative to the calling file.
-        """
-        return self.application.settings.get("template_path")
-
-    @property
-    def xsrf_token(self):
-        """The XSRF-prevention token for the current user/session.
-
-        To prevent cross-site request forgery, we set an '_xsrf' cookie
-        and include the same '_xsrf' value as an argument with all POST
-        requests. If the two do not match, we reject the form submission
-        as a potential forgery.
-
-        See http://en.wikipedia.org/wiki/Cross-site_request_forgery
-
-        .. versionchanged:: 3.2.2
-           The xsrf token will now be have a random mask applied in every
-           request, which makes it safe to include the token in pages
-           that are compressed.  See http://breachattack.com for more
-           information on the issue fixed by this change.  Old (version 1)
-           cookies will be converted to version 2 when this method is called
-           unless the ``xsrf_cookie_version`` `Application` setting is
-           set to 1.
-
-        .. versionchanged:: 4.3
-           The ``xsrf_cookie_kwargs`` `Application` setting may be
-           used to supply additional cookie options (which will be
-           passed directly to `set_cookie`). For example,
-           ``xsrf_cookie_kwargs=dict(httponly=True, secure=True)``
-           will set the ``secure`` and ``httponly`` flags on the
-           ``_xsrf`` cookie.
-        """
-        if not hasattr(self, "_xsrf_token"):
-            version, token, timestamp = self._get_raw_xsrf_token()
-            output_version = self.settings.get("xsrf_cookie_version", 2)
-            cookie_kwargs = self.settings.get("xsrf_cookie_kwargs", {})
-            if output_version == 1:
-                self._xsrf_token = binascii.b2a_hex(token)
-            elif output_version == 2:
-                mask = os.urandom(4)
-                self._xsrf_token = b"|".join([
-                    b"2",
-                    binascii.b2a_hex(mask),
-                    binascii.b2a_hex(_websocket_mask(mask, token)),
-                    utf8(str(int(timestamp)))])
-            else:
-                raise ValueError("unknown xsrf cookie version %d",
-                                 output_version)
-            if version is None:
-                expires_days = 30 if self.current_user else None
-                self.set_cookie("_xsrf", self._xsrf_token,
-                                expires_days=expires_days,
-                                **cookie_kwargs)
-        return self._xsrf_token
-
-    def _get_raw_xsrf_token(self):
-        """Read or generate the xsrf token in its raw form.
-
-        The raw_xsrf_token is a tuple containing:
-
-        * version: the version of the cookie from which this token was read,
-          or None if we generated a new token in this request.
-        * token: the raw token data; random (non-ascii) bytes.
-        * timestamp: the time this token was generated (will not be accurate
-          for version 1 cookies)
-        """
-        if not hasattr(self, '_raw_xsrf_token'):
-            cookie = self.get_cookie("_xsrf")
-            if cookie:
-                version, token, timestamp = self._decode_xsrf_token(cookie)
-            else:
-                version, token, timestamp = None, None, None
-            if token is None:
-                version = None
-                token = os.urandom(16)
-                timestamp = time.time()
-            self._raw_xsrf_token = (version, token, timestamp)
-        return self._raw_xsrf_token
-
-    def _decode_xsrf_token(self, cookie):
-        """Convert a cookie string into a the tuple form returned by
-        _get_raw_xsrf_token.
-        """
-
-        try:
-            m = _signed_value_version_re.match(utf8(cookie))
-
-            if m:
-                version = int(m.group(1))
-                if version == 2:
-                    _, mask, masked_token, timestamp = cookie.split("|")
-
-                    mask = binascii.a2b_hex(utf8(mask))
-                    token = _websocket_mask(
-                        mask, binascii.a2b_hex(utf8(masked_token)))
-                    timestamp = int(timestamp)
-                    return version, token, timestamp
-                else:
-                    # Treat unknown versions as not present instead of failing.
-                    raise Exception("Unknown xsrf cookie version")
-            else:
-                version = 1
-                try:
-                    token = binascii.a2b_hex(utf8(cookie))
-                except (binascii.Error, TypeError):
-                    token = utf8(cookie)
-                # We don't have a usable timestamp in older versions.
-                timestamp = int(time.time())
-                return (version, token, timestamp)
-        except Exception:
-            # Catch exceptions and return nothing instead of failing.
-            gen_log.debug("Uncaught exception in _decode_xsrf_token",
-                          exc_info=True)
-            return None, None, None
-
-    def check_xsrf_cookie(self):
-        """Verifies that the ``_xsrf`` cookie matches the ``_xsrf`` argument.
-
-        To prevent cross-site request forgery, we set an ``_xsrf``
-        cookie and include the same value as a non-cookie
-        field with all ``POST`` requests. If the two do not match, we
-        reject the form submission as a potential forgery.
-
-        The ``_xsrf`` value may be set as either a form field named ``_xsrf``
-        or in a custom HTTP header named ``X-XSRFToken`` or ``X-CSRFToken``
-        (the latter is accepted for compatibility with Django).
-
-        See http://en.wikipedia.org/wiki/Cross-site_request_forgery
-
-        Prior to release 1.1.1, this check was ignored if the HTTP header
-        ``X-Requested-With: XMLHTTPRequest`` was present.  This exception
-        has been shown to be insecure and has been removed.  For more
-        information please see
-        http://www.djangoproject.com/weblog/2011/feb/08/security/
-        http://weblog.rubyonrails.org/2011/2/8/csrf-protection-bypass-in-ruby-on-rails
-
-        .. versionchanged:: 3.2.2
-           Added support for cookie version 2.  Both versions 1 and 2 are
-           supported.
-        """
-        token = (self.get_argument("_xsrf", None) or
-                 self.request.headers.get("X-Xsrftoken") or
-                 self.request.headers.get("X-Csrftoken"))
-        if not token:
-            raise HTTPError(403, "'_xsrf' argument missing from POST")
-        _, token, _ = self._decode_xsrf_token(token)
-        _, expected_token, _ = self._get_raw_xsrf_token()
-        if not token:
-            raise HTTPError(403, "'_xsrf' argument has invalid format")
-        if not _time_independent_equals(utf8(token), utf8(expected_token)):
-            raise HTTPError(403, "XSRF cookie does not match POST argument")
-
-    def xsrf_form_html(self):
-        """An HTML ```` element to be included with all POST forms.
-
-        It defines the ``_xsrf`` input value, which we check on all POST
-        requests to prevent cross-site request forgery. If you have set
-        the ``xsrf_cookies`` application setting, you must include this
-        HTML within all of your HTML forms.
-
-        In a template, this method should be called with ``{% module
-        xsrf_form_html() %}``
-
-        See `check_xsrf_cookie()` above for more information.
-        """
-        return ''
-
-    def static_url(self, path, include_host=None, **kwargs):
-        """Returns a static URL for the given relative static file path.
-
-        This method requires you set the ``static_path`` setting in your
-        application (which specifies the root directory of your static
-        files).
-
-        This method returns a versioned url (by default appending
-        ``?v=``), which allows the static files to be
-        cached indefinitely.  This can be disabled by passing
-        ``include_version=False`` (in the default implementation;
-        other static file implementations are not required to support
-        this, but they may support other options).
-
-        By default this method returns URLs relative to the current
-        host, but if ``include_host`` is true the URL returned will be
-        absolute.  If this handler has an ``include_host`` attribute,
-        that value will be used as the default for all `static_url`
-        calls that do not pass ``include_host`` as a keyword argument.
-
-        """
-        self.require_setting("static_path", "static_url")
-        get_url = self.settings.get("static_handler_class",
-                                    StaticFileHandler).make_static_url
-
-        if include_host is None:
-            include_host = getattr(self, "include_host", False)
-
-        if include_host:
-            base = self.request.protocol + "://" + self.request.host
-        else:
-            base = ""
-
-        return base + get_url(self.settings, path, **kwargs)
-
-    def require_setting(self, name, feature="this feature"):
-        """Raises an exception if the given app setting is not defined."""
-        if not self.application.settings.get(name):
-            raise Exception("You must define the '%s' setting in your "
-                            "application to use %s" % (name, feature))
-
-    def reverse_url(self, name, *args):
-        """Alias for `Application.reverse_url`."""
-        return self.application.reverse_url(name, *args)
-
-    def compute_etag(self):
-        """Computes the etag header to be used for this request.
-
-        By default uses a hash of the content written so far.
-
-        May be overridden to provide custom etag implementations,
-        or may return None to disable tornado's default etag support.
-        """
-        hasher = hashlib.sha1()
-        for part in self._write_buffer:
-            hasher.update(part)
-        return '"%s"' % hasher.hexdigest()
-
-    def set_etag_header(self):
-        """Sets the response's Etag header using ``self.compute_etag()``.
-
-        Note: no header will be set if ``compute_etag()`` returns ``None``.
-
-        This method is called automatically when the request is finished.
-        """
-        etag = self.compute_etag()
-        if etag is not None:
-            self.set_header("Etag", etag)
-
-    def check_etag_header(self):
-        """Checks the ``Etag`` header against requests's ``If-None-Match``.
-
-        Returns ``True`` if the request's Etag matches and a 304 should be
-        returned. For example::
-
-            self.set_etag_header()
-            if self.check_etag_header():
-                self.set_status(304)
-                return
-
-        This method is called automatically when the request is finished,
-        but may be called earlier for applications that override
-        `compute_etag` and want to do an early check for ``If-None-Match``
-        before completing the request.  The ``Etag`` header should be set
-        (perhaps with `set_etag_header`) before calling this method.
-        """
-        computed_etag = utf8(self._headers.get("Etag", ""))
-        # Find all weak and strong etag values from If-None-Match header
-        # because RFC 7232 allows multiple etag values in a single header.
-        etags = re.findall(
-            br'\*|(?:W/)?"[^"]*"',
-            utf8(self.request.headers.get("If-None-Match", ""))
-        )
-        if not computed_etag or not etags:
-            return False
-
-        match = False
-        if etags[0] == b'*':
-            match = True
-        else:
-            # Use a weak comparison when comparing entity-tags.
-            def val(x):
-                return x[2:] if x.startswith(b'W/') else x
-
-            for etag in etags:
-                if val(etag) == val(computed_etag):
-                    match = True
-                    break
-        return match
-
-    def _stack_context_handle_exception(self, type, value, traceback):
-        try:
-            # For historical reasons _handle_request_exception only takes
-            # the exception value instead of the full triple,
-            # so re-raise the exception to ensure that it's in
-            # sys.exc_info()
-            raise_exc_info((type, value, traceback))
-        except Exception:
-            self._handle_request_exception(value)
-        return True
-
-    @gen.coroutine
-    def _execute(self, transforms, *args, **kwargs):
-        """Executes this request with the given output transforms."""
-        self._transforms = transforms
-        try:
-            if self.request.method not in self.SUPPORTED_METHODS:
-                raise HTTPError(405)
-            self.path_args = [self.decode_argument(arg) for arg in args]
-            self.path_kwargs = dict((k, self.decode_argument(v, name=k))
-                                    for (k, v) in kwargs.items())
-            # If XSRF cookies are turned on, reject form submissions without
-            # the proper cookie
-            if self.request.method not in ("GET", "HEAD", "OPTIONS") and \
-                    self.application.settings.get("xsrf_cookies"):
-                self.check_xsrf_cookie()
-
-            result = self.prepare()
-            if result is not None:
-                result = yield result
-            if self._prepared_future is not None:
-                # Tell the Application we've finished with prepare()
-                # and are ready for the body to arrive.
-                self._prepared_future.set_result(None)
-            if self._finished:
-                return
-
-            if _has_stream_request_body(self.__class__):
-                # In streaming mode request.body is a Future that signals
-                # the body has been completely received.  The Future has no
-                # result; the data has been passed to self.data_received
-                # instead.
-                try:
-                    yield self.request.body
-                except iostream.StreamClosedError:
-                    return
-
-            method = getattr(self, self.request.method.lower())
-            result = method(*self.path_args, **self.path_kwargs)
-            if result is not None:
-                result = yield result
-            if self._auto_finish and not self._finished:
-                self.finish()
-        except Exception as e:
-            try:
-                self._handle_request_exception(e)
-            except Exception:
-                app_log.error("Exception in exception handler", exc_info=True)
-            if (self._prepared_future is not None and
-                    not self._prepared_future.done()):
-                # In case we failed before setting _prepared_future, do it
-                # now (to unblock the HTTP server).  Note that this is not
-                # in a finally block to avoid GC issues prior to Python 3.4.
-                self._prepared_future.set_result(None)
-
-    def data_received(self, chunk):
-        """Implement this method to handle streamed request data.
-
-        Requires the `.stream_request_body` decorator.
-        """
-        raise NotImplementedError()
-
-    def _log(self):
-        """Logs the current request.
-
-        Sort of deprecated since this functionality was moved to the
-        Application, but left in place for the benefit of existing apps
-        that have overridden this method.
-        """
-        self.application.log_request(self)
-
-    def _request_summary(self):
-        return "%s %s (%s)" % (self.request.method, self.request.uri,
-                               self.request.remote_ip)
-
-    def _handle_request_exception(self, e):
-        if isinstance(e, Finish):
-            # Not an error; just finish the request without logging.
-            if not self._finished:
-                self.finish(*e.args)
-            return
-        try:
-            self.log_exception(*sys.exc_info())
-        except Exception:
-            # An error here should still get a best-effort send_error()
-            # to avoid leaking the connection.
-            app_log.error("Error in exception logger", exc_info=True)
-        if self._finished:
-            # Extra errors after the request has been finished should
-            # be logged, but there is no reason to continue to try and
-            # send a response.
-            return
-        if isinstance(e, HTTPError):
-            if e.status_code not in httputil.responses and not e.reason:
-                gen_log.error("Bad HTTP status code: %d", e.status_code)
-                self.send_error(500, exc_info=sys.exc_info())
-            else:
-                self.send_error(e.status_code, exc_info=sys.exc_info())
-        else:
-            self.send_error(500, exc_info=sys.exc_info())
-
-    def log_exception(self, typ, value, tb):
-        """Override to customize logging of uncaught exceptions.
-
-        By default logs instances of `HTTPError` as warnings without
-        stack traces (on the ``tornado.general`` logger), and all
-        other exceptions as errors with stack traces (on the
-        ``tornado.application`` logger).
-
-        .. versionadded:: 3.1
-        """
-        if isinstance(value, HTTPError):
-            if value.log_message:
-                format = "%d %s: " + value.log_message
-                args = ([value.status_code, self._request_summary()] +
-                        list(value.args))
-                gen_log.warning(format, *args)
-        else:
-            app_log.error("Uncaught exception %s\n%r", self._request_summary(),
-                          self.request, exc_info=(typ, value, tb))
-
-    def _ui_module(self, name, module):
-        def render(*args, **kwargs):
-            if not hasattr(self, "_active_modules"):
-                self._active_modules = {}
-            if name not in self._active_modules:
-                self._active_modules[name] = module(self)
-            rendered = self._active_modules[name].render(*args, **kwargs)
-            return rendered
-        return render
-
-    def _ui_method(self, method):
-        return lambda *args, **kwargs: method(self, *args, **kwargs)
-
-    def _clear_headers_for_304(self):
-        # 304 responses should not contain entity headers (defined in
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec7.html#sec7.1)
-        # not explicitly allowed by
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5
-        headers = ["Allow", "Content-Encoding", "Content-Language",
-                   "Content-Length", "Content-MD5", "Content-Range",
-                   "Content-Type", "Last-Modified"]
-        for h in headers:
-            self.clear_header(h)
-
-
-def asynchronous(method):
-    """Wrap request handler methods with this if they are asynchronous.
-
-    This decorator is for callback-style asynchronous methods; for
-    coroutines, use the ``@gen.coroutine`` decorator without
-    ``@asynchronous``. (It is legal for legacy reasons to use the two
-    decorators together provided ``@asynchronous`` is first, but
-    ``@asynchronous`` will be ignored in this case)
-
-    This decorator should only be applied to the :ref:`HTTP verb
-    methods `; its behavior is undefined for any other method.
-    This decorator does not *make* a method asynchronous; it tells
-    the framework that the method *is* asynchronous.  For this decorator
-    to be useful the method must (at least sometimes) do something
-    asynchronous.
-
-    If this decorator is given, the response is not finished when the
-    method returns. It is up to the request handler to call
-    `self.finish() ` to finish the HTTP
-    request. Without this decorator, the request is automatically
-    finished when the ``get()`` or ``post()`` method returns. Example:
-
-    .. testcode::
-
-       class MyRequestHandler(RequestHandler):
-           @asynchronous
-           def get(self):
-              http = httpclient.AsyncHTTPClient()
-              http.fetch("http://friendfeed.com/", self._on_download)
-
-           def _on_download(self, response):
-              self.write("Downloaded!")
-              self.finish()
-
-    .. testoutput::
-       :hide:
-
-    .. versionchanged:: 3.1
-       The ability to use ``@gen.coroutine`` without ``@asynchronous``.
-
-    .. versionchanged:: 4.3 Returning anything but ``None`` or a
-       yieldable object from a method decorated with ``@asynchronous``
-       is an error. Such return values were previously ignored silently.
-    """
-    # Delay the IOLoop import because it's not available on app engine.
-    from salt.ext.tornado.ioloop import IOLoop
-
-    @functools.wraps(method)
-    def wrapper(self, *args, **kwargs):
-        self._auto_finish = False
-        with stack_context.ExceptionStackContext(
-                self._stack_context_handle_exception):
-            result = method(self, *args, **kwargs)
-            if result is not None:
-                result = gen.convert_yielded(result)
-
-                # If @asynchronous is used with @gen.coroutine, (but
-                # not @gen.engine), we can automatically finish the
-                # request when the future resolves.  Additionally,
-                # the Future will swallow any exceptions so we need
-                # to throw them back out to the stack context to finish
-                # the request.
-                def future_complete(f):
-                    f.result()
-                    if not self._finished:
-                        self.finish()
-                IOLoop.current().add_future(result, future_complete)
-                # Once we have done this, hide the Future from our
-                # caller (i.e. RequestHandler._when_complete), which
-                # would otherwise set up its own callback and
-                # exception handler (resulting in exceptions being
-                # logged twice).
-                return None
-            return result
-    return wrapper
-
-
-def stream_request_body(cls):
-    """Apply to `RequestHandler` subclasses to enable streaming body support.
-
-    This decorator implies the following changes:
-
-    * `.HTTPServerRequest.body` is undefined, and body arguments will not
-      be included in `RequestHandler.get_argument`.
-    * `RequestHandler.prepare` is called when the request headers have been
-      read instead of after the entire body has been read.
-    * The subclass must define a method ``data_received(self, data):``, which
-      will be called zero or more times as data is available.  Note that
-      if the request has an empty body, ``data_received`` may not be called.
-    * ``prepare`` and ``data_received`` may return Futures (such as via
-      ``@gen.coroutine``, in which case the next method will not be called
-      until those futures have completed.
-    * The regular HTTP method (``post``, ``put``, etc) will be called after
-      the entire body has been read.
-
-    See the `file receiver demo `_
-    for example usage.
-    """
-    if not issubclass(cls, RequestHandler):
-        raise TypeError("expected subclass of RequestHandler, got %r", cls)
-    cls._stream_request_body = True
-    return cls
-
-
-def _has_stream_request_body(cls):
-    if not issubclass(cls, RequestHandler):
-        raise TypeError("expected subclass of RequestHandler, got %r", cls)
-    return getattr(cls, '_stream_request_body', False)
-
-
-def removeslash(method):
-    """Use this decorator to remove trailing slashes from the request path.
-
-    For example, a request to ``/foo/`` would redirect to ``/foo`` with this
-    decorator. Your request handler mapping should use a regular expression
-    like ``r'/foo/*'`` in conjunction with using the decorator.
-    """
-    @functools.wraps(method)
-    def wrapper(self, *args, **kwargs):
-        if self.request.path.endswith("/"):
-            if self.request.method in ("GET", "HEAD"):
-                uri = self.request.path.rstrip("/")
-                if uri:  # don't try to redirect '/' to ''
-                    if self.request.query:
-                        uri += "?" + self.request.query
-                    self.redirect(uri, permanent=True)
-                    return
-            else:
-                raise HTTPError(404)
-        return method(self, *args, **kwargs)
-    return wrapper
-
-
-def addslash(method):
-    """Use this decorator to add a missing trailing slash to the request path.
-
-    For example, a request to ``/foo`` would redirect to ``/foo/`` with this
-    decorator. Your request handler mapping should use a regular expression
-    like ``r'/foo/?'`` in conjunction with using the decorator.
-    """
-    @functools.wraps(method)
-    def wrapper(self, *args, **kwargs):
-        if not self.request.path.endswith("/"):
-            if self.request.method in ("GET", "HEAD"):
-                uri = self.request.path + "/"
-                if self.request.query:
-                    uri += "?" + self.request.query
-                self.redirect(uri, permanent=True)
-                return
-            raise HTTPError(404)
-        return method(self, *args, **kwargs)
-    return wrapper
-
-
-class _ApplicationRouter(ReversibleRuleRouter):
-    """Routing implementation used internally by `Application`.
-
-    Provides a binding between `Application` and `RequestHandler`.
-    This implementation extends `~.routing.ReversibleRuleRouter` in a couple of ways:
-        * it allows to use `RequestHandler` subclasses as `~.routing.Rule` target and
-        * it allows to use a list/tuple of rules as `~.routing.Rule` target.
-        ``process_rule`` implementation will substitute this list with an appropriate
-        `_ApplicationRouter` instance.
-    """
-
-    def __init__(self, application, rules=None):
-        assert isinstance(application, Application)
-        self.application = application
-        super(_ApplicationRouter, self).__init__(rules)
-
-    def process_rule(self, rule):
-        rule = super(_ApplicationRouter, self).process_rule(rule)
-
-        if isinstance(rule.target, (list, tuple)):
-            rule.target = _ApplicationRouter(self.application, rule.target)
-
-        return rule
-
-    def get_target_delegate(self, target, request, **target_params):
-        if isclass(target) and issubclass(target, RequestHandler):
-            return self.application.get_handler_delegate(request, target, **target_params)
-
-        return super(_ApplicationRouter, self).get_target_delegate(target, request, **target_params)
-
-
-class Application(ReversibleRouter):
-    r"""A collection of request handlers that make up a web application.
-
-    Instances of this class are callable and can be passed directly to
-    HTTPServer to serve the application::
-
-        application = web.Application([
-            (r"/", MainPageHandler),
-        ])
-        http_server = httpserver.HTTPServer(application)
-        http_server.listen(8080)
-        ioloop.IOLoop.current().start()
-
-    The constructor for this class takes in a list of `~.routing.Rule`
-    objects or tuples of values corresponding to the arguments of
-    `~.routing.Rule` constructor: ``(matcher, target, [target_kwargs], [name])``,
-    the values in square brackets being optional. The default matcher is
-    `~.routing.PathMatches`, so ``(regexp, target)`` tuples can also be used
-    instead of ``(PathMatches(regexp), target)``.
-
-    A common routing target is a `RequestHandler` subclass, but you can also
-    use lists of rules as a target, which create a nested routing configuration::
-
-        application = web.Application([
-            (HostMatches("example.com"), [
-                (r"/", MainPageHandler),
-                (r"/feed", FeedHandler),
-            ]),
-        ])
-
-    In addition to this you can use nested `~.routing.Router` instances,
-    `~.httputil.HTTPMessageDelegate` subclasses and callables as routing targets
-    (see `~.routing` module docs for more information).
-
-    When we receive requests, we iterate over the list in order and
-    instantiate an instance of the first request class whose regexp
-    matches the request path. The request class can be specified as
-    either a class object or a (fully-qualified) name.
-
-    A dictionary may be passed as the third element (``target_kwargs``)
-    of the tuple, which will be used as keyword arguments to the handler's
-    constructor and `~RequestHandler.initialize` method. This pattern
-    is used for the `StaticFileHandler` in this example (note that a
-    `StaticFileHandler` can be installed automatically with the
-    static_path setting described below)::
-
-        application = web.Application([
-            (r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
-        ])
-
-    We support virtual hosts with the `add_handlers` method, which takes in
-    a host regular expression as the first argument::
-
-        application.add_handlers(r"www\.myhost\.com", [
-            (r"/article/([0-9]+)", ArticleHandler),
-        ])
-
-    If there's no match for the current request's host, then ``default_host``
-    parameter value is matched against host regular expressions.
-
-    You can serve static files by sending the ``static_path`` setting
-    as a keyword argument. We will serve those files from the
-    ``/static/`` URI (this is configurable with the
-    ``static_url_prefix`` setting), and we will serve ``/favicon.ico``
-    and ``/robots.txt`` from the same directory.  A custom subclass of
-    `StaticFileHandler` can be specified with the
-    ``static_handler_class`` setting.
-
-    .. versionchanged:: 4.5
-       Integration with the new `tornado.routing` module.
-    """
-    def __init__(self, handlers=None, default_host=None, transforms=None,
-                 **settings):
-        if transforms is None:
-            self.transforms = []
-            if settings.get("compress_response") or settings.get("gzip"):
-                self.transforms.append(GZipContentEncoding)
-        else:
-            self.transforms = transforms
-        self.default_host = default_host
-        self.settings = settings
-        self.ui_modules = {'linkify': _linkify,
-                           'xsrf_form_html': _xsrf_form_html,
-                           'Template': TemplateModule,
-                           }
-        self.ui_methods = {}
-        self._load_ui_modules(settings.get("ui_modules", {}))
-        self._load_ui_methods(settings.get("ui_methods", {}))
-        if self.settings.get("static_path"):
-            path = self.settings["static_path"]
-            handlers = list(handlers or [])
-            static_url_prefix = settings.get("static_url_prefix",
-                                             "/static/")
-            static_handler_class = settings.get("static_handler_class",
-                                                StaticFileHandler)
-            static_handler_args = settings.get("static_handler_args", {})
-            static_handler_args['path'] = path
-            for pattern in [re.escape(static_url_prefix) + r"(.*)",
-                            r"/(favicon\.ico)", r"/(robots\.txt)"]:
-                handlers.insert(0, (pattern, static_handler_class,
-                                    static_handler_args))
-
-        if self.settings.get('debug'):
-            self.settings.setdefault('autoreload', True)
-            self.settings.setdefault('compiled_template_cache', False)
-            self.settings.setdefault('static_hash_cache', False)
-            self.settings.setdefault('serve_traceback', True)
-
-        self.wildcard_router = _ApplicationRouter(self, handlers)
-        self.default_router = _ApplicationRouter(self, [
-            Rule(AnyMatches(), self.wildcard_router)
-        ])
-
-        # Automatically reload modified modules
-        if self.settings.get('autoreload'):
-            from salt.ext.tornado import autoreload
-            autoreload.start()
-
-    def listen(self, port, address="", **kwargs):
-        """Starts an HTTP server for this application on the given port.
-
-        This is a convenience alias for creating an `.HTTPServer`
-        object and calling its listen method.  Keyword arguments not
-        supported by `HTTPServer.listen <.TCPServer.listen>` are passed to the
-        `.HTTPServer` constructor.  For advanced uses
-        (e.g. multi-process mode), do not use this method; create an
-        `.HTTPServer` and call its
-        `.TCPServer.bind`/`.TCPServer.start` methods directly.
-
-        Note that after calling this method you still need to call
-        ``IOLoop.current().start()`` to start the server.
-
-        Returns the `.HTTPServer` object.
-
-        .. versionchanged:: 4.3
-           Now returns the `.HTTPServer` object.
-        """
-        # import is here rather than top level because HTTPServer
-        # is not importable on appengine
-        from salt.ext.tornado.httpserver import HTTPServer
-        server = HTTPServer(self, **kwargs)
-        server.listen(port, address)
-        return server
-
-    def add_handlers(self, host_pattern, host_handlers):
-        """Appends the given handlers to our handler list.
-
-        Host patterns are processed sequentially in the order they were
-        added. All matching patterns will be considered.
-        """
-        host_matcher = HostMatches(host_pattern)
-        rule = Rule(host_matcher, _ApplicationRouter(self, host_handlers))
-
-        self.default_router.rules.insert(-1, rule)
-
-        if self.default_host is not None:
-            self.wildcard_router.add_rules([(
-                DefaultHostMatches(self, host_matcher.host_pattern),
-                host_handlers
-            )])
-
-    def add_transform(self, transform_class):
-        self.transforms.append(transform_class)
-
-    def _load_ui_methods(self, methods):
-        if isinstance(methods, types.ModuleType):
-            self._load_ui_methods(dict((n, getattr(methods, n))
-                                       for n in dir(methods)))
-        elif isinstance(methods, list):
-            for m in methods:
-                self._load_ui_methods(m)
-        else:
-            for name, fn in methods.items():
-                if not name.startswith("_") and hasattr(fn, "__call__") \
-                        and name[0].lower() == name[0]:
-                    self.ui_methods[name] = fn
-
-    def _load_ui_modules(self, modules):
-        if isinstance(modules, types.ModuleType):
-            self._load_ui_modules(dict((n, getattr(modules, n))
-                                       for n in dir(modules)))
-        elif isinstance(modules, list):
-            for m in modules:
-                self._load_ui_modules(m)
-        else:
-            assert isinstance(modules, dict)
-            for name, cls in modules.items():
-                try:
-                    if issubclass(cls, UIModule):
-                        self.ui_modules[name] = cls
-                except TypeError:
-                    pass
-
-    def __call__(self, request):
-        # Legacy HTTPServer interface
-        dispatcher = self.find_handler(request)
-        return dispatcher.execute()
-
-    def find_handler(self, request, **kwargs):
-        route = self.default_router.find_handler(request)
-        if route is not None:
-            return route
-
-        if self.settings.get('default_handler_class'):
-            return self.get_handler_delegate(
-                request,
-                self.settings['default_handler_class'],
-                self.settings.get('default_handler_args', {}))
-
-        return self.get_handler_delegate(
-            request, ErrorHandler, {'status_code': 404})
-
-    def get_handler_delegate(self, request, target_class, target_kwargs=None,
-                             path_args=None, path_kwargs=None):
-        """Returns `~.httputil.HTTPMessageDelegate` that can serve a request
-        for application and `RequestHandler` subclass.
-
-        :arg httputil.HTTPServerRequest request: current HTTP request.
-        :arg RequestHandler target_class: a `RequestHandler` class.
-        :arg dict target_kwargs: keyword arguments for ``target_class`` constructor.
-        :arg list path_args: positional arguments for ``target_class`` HTTP method that
-            will be executed while handling a request (``get``, ``post`` or any other).
-        :arg dict path_kwargs: keyword arguments for ``target_class`` HTTP method.
-        """
-        return _HandlerDelegate(
-            self, request, target_class, target_kwargs, path_args, path_kwargs)
-
-    def reverse_url(self, name, *args):
-        """Returns a URL path for handler named ``name``
-
-        The handler must be added to the application as a named `URLSpec`.
-
-        Args will be substituted for capturing groups in the `URLSpec` regex.
-        They will be converted to strings if necessary, encoded as utf8,
-        and url-escaped.
-        """
-        reversed_url = self.default_router.reverse_url(name, *args)
-        if reversed_url is not None:
-            return reversed_url
-
-        raise KeyError("%s not found in named urls" % name)
-
-    def log_request(self, handler):
-        """Writes a completed HTTP request to the logs.
-
-        By default writes to the python root logger.  To change
-        this behavior either subclass Application and override this method,
-        or pass a function in the application settings dictionary as
-        ``log_function``.
-        """
-        if "log_function" in self.settings:
-            self.settings["log_function"](handler)
-            return
-        if handler.get_status() < 400:
-            log_method = access_log.info
-        elif handler.get_status() < 500:
-            log_method = access_log.warning
-        else:
-            log_method = access_log.error
-        request_time = 1000.0 * handler.request.request_time()
-        log_method("%d %s %.2fms", handler.get_status(),
-                   handler._request_summary(), request_time)
-
-
-class _HandlerDelegate(httputil.HTTPMessageDelegate):
-    def __init__(self, application, request, handler_class, handler_kwargs,
-                 path_args, path_kwargs):
-        self.application = application
-        self.connection = request.connection
-        self.request = request
-        self.handler_class = handler_class
-        self.handler_kwargs = handler_kwargs or {}
-        self.path_args = path_args or []
-        self.path_kwargs = path_kwargs or {}
-        self.chunks = []
-        self.stream_request_body = _has_stream_request_body(self.handler_class)
-
-    def headers_received(self, start_line, headers):
-        if self.stream_request_body:
-            self.request.body = Future()
-            return self.execute()
-
-    def data_received(self, data):
-        if self.stream_request_body:
-            return self.handler.data_received(data)
-        else:
-            self.chunks.append(data)
-
-    def finish(self):
-        if self.stream_request_body:
-            self.request.body.set_result(None)
-        else:
-            self.request.body = b''.join(self.chunks)
-            self.request._parse_body()
-            self.execute()
-
-    def on_connection_close(self):
-        if self.stream_request_body:
-            self.handler.on_connection_close()
-        else:
-            self.chunks = None
-
-    def execute(self):
-        # If template cache is disabled (usually in the debug mode),
-        # re-compile templates and reload static files on every
-        # request so you don't need to restart to see changes
-        if not self.application.settings.get("compiled_template_cache", True):
-            with RequestHandler._template_loader_lock:
-                for loader in RequestHandler._template_loaders.values():
-                    loader.reset()
-        if not self.application.settings.get('static_hash_cache', True):
-            StaticFileHandler.reset()
-
-        self.handler = self.handler_class(self.application, self.request,
-                                          **self.handler_kwargs)
-        transforms = [t(self.request) for t in self.application.transforms]
-
-        if self.stream_request_body:
-            self.handler._prepared_future = Future()
-        # Note that if an exception escapes handler._execute it will be
-        # trapped in the Future it returns (which we are ignoring here,
-        # leaving it to be logged when the Future is GC'd).
-        # However, that shouldn't happen because _execute has a blanket
-        # except handler, and we cannot easily access the IOLoop here to
-        # call add_future (because of the requirement to remain compatible
-        # with WSGI)
-        self.handler._execute(transforms, *self.path_args,
-                              **self.path_kwargs)
-        # If we are streaming the request body, then execute() is finished
-        # when the handler has prepared to receive the body.  If not,
-        # it doesn't matter when execute() finishes (so we return None)
-        return self.handler._prepared_future
-
-
-class HTTPError(Exception):
-    """An exception that will turn into an HTTP error response.
-
-    Raising an `HTTPError` is a convenient alternative to calling
-    `RequestHandler.send_error` since it automatically ends the
-    current function.
-
-    To customize the response sent with an `HTTPError`, override
-    `RequestHandler.write_error`.
-
-    :arg int status_code: HTTP status code.  Must be listed in
-        `httplib.responses ` unless the ``reason``
-        keyword argument is given.
-    :arg string log_message: Message to be written to the log for this error
-        (will not be shown to the user unless the `Application` is in debug
-        mode).  May contain ``%s``-style placeholders, which will be filled
-        in with remaining positional parameters.
-    :arg string reason: Keyword-only argument.  The HTTP "reason" phrase
-        to pass in the status line along with ``status_code``.  Normally
-        determined automatically from ``status_code``, but can be used
-        to use a non-standard numeric code.
-    """
-    def __init__(self, status_code=500, log_message=None, *args, **kwargs):
-        self.status_code = status_code
-        self.log_message = log_message
-        self.args = args
-        self.reason = kwargs.get('reason', None)
-        if log_message and not args:
-            self.log_message = log_message.replace('%', '%%')
-
-    def __str__(self):
-        message = "HTTP %d: %s" % (
-            self.status_code,
-            self.reason or httputil.responses.get(self.status_code, 'Unknown'))
-        if self.log_message:
-            return message + " (" + (self.log_message % self.args) + ")"
-        else:
-            return message
-
-
-class Finish(Exception):
-    """An exception that ends the request without producing an error response.
-
-    When `Finish` is raised in a `RequestHandler`, the request will
-    end (calling `RequestHandler.finish` if it hasn't already been
-    called), but the error-handling methods (including
-    `RequestHandler.write_error`) will not be called.
-
-    If `Finish()` was created with no arguments, the pending response
-    will be sent as-is. If `Finish()` was given an argument, that
-    argument will be passed to `RequestHandler.finish()`.
-
-    This can be a more convenient way to implement custom error pages
-    than overriding ``write_error`` (especially in library code)::
-
-        if self.current_user is None:
-            self.set_status(401)
-            self.set_header('WWW-Authenticate', 'Basic realm="something"')
-            raise Finish()
-
-    .. versionchanged:: 4.3
-       Arguments passed to ``Finish()`` will be passed on to
-       `RequestHandler.finish`.
-    """
-    pass
-
-
-class MissingArgumentError(HTTPError):
-    """Exception raised by `RequestHandler.get_argument`.
-
-    This is a subclass of `HTTPError`, so if it is uncaught a 400 response
-    code will be used instead of 500 (and a stack trace will not be logged).
-
-    .. versionadded:: 3.1
-    """
-    def __init__(self, arg_name):
-        super(MissingArgumentError, self).__init__(
-            400, 'Missing argument %s' % arg_name)
-        self.arg_name = arg_name
-
-
-class ErrorHandler(RequestHandler):
-    """Generates an error response with ``status_code`` for all requests."""
-    def initialize(self, status_code):
-        self.set_status(status_code)
-
-    def prepare(self):
-        raise HTTPError(self._status_code)
-
-    def check_xsrf_cookie(self):
-        # POSTs to an ErrorHandler don't actually have side effects,
-        # so we don't need to check the xsrf token.  This allows POSTs
-        # to the wrong url to return a 404 instead of 403.
-        pass
-
-
-class RedirectHandler(RequestHandler):
-    """Redirects the client to the given URL for all GET requests.
-
-    You should provide the keyword argument ``url`` to the handler, e.g.::
-
-        application = web.Application([
-            (r"/oldpath", web.RedirectHandler, {"url": "/newpath"}),
-        ])
-
-    `RedirectHandler` supports regular expression substitutions. E.g., to
-    swap the first and second parts of a path while preserving the remainder::
-
-        application = web.Application([
-            (r"/(.*?)/(.*?)/(.*)", web.RedirectHandler, {"url": "/{1}/{0}/{2}"}),
-        ])
-
-    The final URL is formatted with `str.format` and the substrings that match
-    the capturing groups. In the above example, a request to "/a/b/c" would be
-    formatted like::
-
-        str.format("/{1}/{0}/{2}", "a", "b", "c")  # -> "/b/a/c"
-
-    Use Python's :ref:`format string syntax ` to customize how
-    values are substituted.
-
-    .. versionchanged:: 4.5
-       Added support for substitutions into the destination URL.
-    """
-    def initialize(self, url, permanent=True):
-        self._url = url
-        self._permanent = permanent
-
-    def get(self, *args):
-        self.redirect(self._url.format(*args), permanent=self._permanent)
-
-
-class StaticFileHandler(RequestHandler):
-    """A simple handler that can serve static content from a directory.
-
-    A `StaticFileHandler` is configured automatically if you pass the
-    ``static_path`` keyword argument to `Application`.  This handler
-    can be customized with the ``static_url_prefix``, ``static_handler_class``,
-    and ``static_handler_args`` settings.
-
-    To map an additional path to this handler for a static data directory
-    you would add a line to your application like::
-
-        application = web.Application([
-            (r"/content/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
-        ])
-
-    The handler constructor requires a ``path`` argument, which specifies the
-    local root directory of the content to be served.
-
-    Note that a capture group in the regex is required to parse the value for
-    the ``path`` argument to the get() method (different than the constructor
-    argument above); see `URLSpec` for details.
-
-    To serve a file like ``index.html`` automatically when a directory is
-    requested, set ``static_handler_args=dict(default_filename="index.html")``
-    in your application settings, or add ``default_filename`` as an initializer
-    argument for your ``StaticFileHandler``.
-
-    To maximize the effectiveness of browser caching, this class supports
-    versioned urls (by default using the argument ``?v=``).  If a version
-    is given, we instruct the browser to cache this file indefinitely.
-    `make_static_url` (also available as `RequestHandler.static_url`) can
-    be used to construct a versioned url.
-
-    This handler is intended primarily for use in development and light-duty
-    file serving; for heavy traffic it will be more efficient to use
-    a dedicated static file server (such as nginx or Apache).  We support
-    the HTTP ``Accept-Ranges`` mechanism to return partial content (because
-    some browsers require this functionality to be present to seek in
-    HTML5 audio or video).
-
-    **Subclassing notes**
-
-    This class is designed to be extensible by subclassing, but because
-    of the way static urls are generated with class methods rather than
-    instance methods, the inheritance patterns are somewhat unusual.
-    Be sure to use the ``@classmethod`` decorator when overriding a
-    class method.  Instance methods may use the attributes ``self.path``
-    ``self.absolute_path``, and ``self.modified``.
-
-    Subclasses should only override methods discussed in this section;
-    overriding other methods is error-prone.  Overriding
-    ``StaticFileHandler.get`` is particularly problematic due to the
-    tight coupling with ``compute_etag`` and other methods.
-
-    To change the way static urls are generated (e.g. to match the behavior
-    of another server or CDN), override `make_static_url`, `parse_url_path`,
-    `get_cache_time`, and/or `get_version`.
-
-    To replace all interaction with the filesystem (e.g. to serve
-    static content from a database), override `get_content`,
-    `get_content_size`, `get_modified_time`, `get_absolute_path`, and
-    `validate_absolute_path`.
-
-    .. versionchanged:: 3.1
-       Many of the methods for subclasses were added in Tornado 3.1.
-    """
-    CACHE_MAX_AGE = 86400 * 365 * 10  # 10 years
-
-    _static_hashes = {}  # type: typing.Dict
-    _lock = threading.Lock()  # protects _static_hashes
-
-    def initialize(self, path, default_filename=None):
-        self.root = path
-        self.default_filename = default_filename
-
-    @classmethod
-    def reset(cls):
-        with cls._lock:
-            cls._static_hashes = {}
-
-    def head(self, path):
-        return self.get(path, include_body=False)
-
-    @gen.coroutine
-    def get(self, path, include_body=True):
-        # Set up our path instance variables.
-        self.path = self.parse_url_path(path)
-        del path  # make sure we don't refer to path instead of self.path again
-        absolute_path = self.get_absolute_path(self.root, self.path)
-        self.absolute_path = self.validate_absolute_path(
-            self.root, absolute_path)
-        if self.absolute_path is None:
-            return
-
-        self.modified = self.get_modified_time()
-        self.set_headers()
-
-        if self.should_return_304():
-            self.set_status(304)
-            return
-
-        request_range = None
-        range_header = self.request.headers.get("Range")
-        if range_header:
-            # As per RFC 2616 14.16, if an invalid Range header is specified,
-            # the request will be treated as if the header didn't exist.
-            request_range = httputil._parse_request_range(range_header)
-
-        size = self.get_content_size()
-        if request_range:
-            start, end = request_range
-            if (start is not None and start >= size) or end == 0:
-                # As per RFC 2616 14.35.1, a range is not satisfiable only: if
-                # the first requested byte is equal to or greater than the
-                # content, or when a suffix with length 0 is specified
-                self.set_status(416)  # Range Not Satisfiable
-                self.set_header("Content-Type", "text/plain")
-                self.set_header("Content-Range", "bytes */%s" % (size, ))
-                return
-            if start is not None and start < 0:
-                start += size
-            if end is not None and end > size:
-                # Clients sometimes blindly use a large range to limit their
-                # download size; cap the endpoint at the actual file size.
-                end = size
-            # Note: only return HTTP 206 if less than the entire range has been
-            # requested. Not only is this semantically correct, but Chrome
-            # refuses to play audio if it gets an HTTP 206 in response to
-            # ``Range: bytes=0-``.
-            if size != (end or size) - (start or 0):
-                self.set_status(206)  # Partial Content
-                self.set_header("Content-Range",
-                                httputil._get_content_range(start, end, size))
-        else:
-            start = end = None
-
-        if start is not None and end is not None:
-            content_length = end - start
-        elif end is not None:
-            content_length = end
-        elif start is not None:
-            content_length = size - start
-        else:
-            content_length = size
-        self.set_header("Content-Length", content_length)
-
-        if include_body:
-            content = self.get_content(self.absolute_path, start, end)
-            if isinstance(content, bytes):
-                content = [content]
-            for chunk in content:
-                try:
-                    self.write(chunk)
-                    yield self.flush()
-                except iostream.StreamClosedError:
-                    return
-        else:
-            assert self.request.method == "HEAD"
-
-    def compute_etag(self):
-        """Sets the ``Etag`` header based on static url version.
-
-        This allows efficient ``If-None-Match`` checks against cached
-        versions, and sends the correct ``Etag`` for a partial response
-        (i.e. the same ``Etag`` as the full file).
-
-        .. versionadded:: 3.1
-        """
-        version_hash = self._get_cached_version(self.absolute_path)
-        if not version_hash:
-            return None
-        return '"%s"' % (version_hash, )
-
-    def set_headers(self):
-        """Sets the content and caching headers on the response.
-
-        .. versionadded:: 3.1
-        """
-        self.set_header("Accept-Ranges", "bytes")
-        self.set_etag_header()
-
-        if self.modified is not None:
-            self.set_header("Last-Modified", self.modified)
-
-        content_type = self.get_content_type()
-        if content_type:
-            self.set_header("Content-Type", content_type)
-
-        cache_time = self.get_cache_time(self.path, self.modified,
-                                         content_type)
-        if cache_time > 0:
-            self.set_header("Expires", datetime.datetime.utcnow() +
-                            datetime.timedelta(seconds=cache_time))
-            self.set_header("Cache-Control", "max-age=" + str(cache_time))
-
-        self.set_extra_headers(self.path)
-
-    def should_return_304(self):
-        """Returns True if the headers indicate that we should return 304.
-
-        .. versionadded:: 3.1
-        """
-        if self.check_etag_header():
-            return True
-
-        # Check the If-Modified-Since, and don't send the result if the
-        # content has not been modified
-        ims_value = self.request.headers.get("If-Modified-Since")
-        if ims_value is not None:
-            date_tuple = email.utils.parsedate(ims_value)
-            if date_tuple is not None:
-                if_since = datetime.datetime(*date_tuple[:6])
-                if if_since >= self.modified:
-                    return True
-
-        return False
-
-    @classmethod
-    def get_absolute_path(cls, root, path):
-        """Returns the absolute location of ``path`` relative to ``root``.
-
-        ``root`` is the path configured for this `StaticFileHandler`
-        (in most cases the ``static_path`` `Application` setting).
-
-        This class method may be overridden in subclasses.  By default
-        it returns a filesystem path, but other strings may be used
-        as long as they are unique and understood by the subclass's
-        overridden `get_content`.
-
-        .. versionadded:: 3.1
-        """
-        abspath = os.path.abspath(os.path.join(root, path))
-        return abspath
-
-    def validate_absolute_path(self, root, absolute_path):
-        """Validate and return the absolute path.
-
-        ``root`` is the configured path for the `StaticFileHandler`,
-        and ``path`` is the result of `get_absolute_path`
-
-        This is an instance method called during request processing,
-        so it may raise `HTTPError` or use methods like
-        `RequestHandler.redirect` (return None after redirecting to
-        halt further processing).  This is where 404 errors for missing files
-        are generated.
-
-        This method may modify the path before returning it, but note that
-        any such modifications will not be understood by `make_static_url`.
-
-        In instance methods, this method's result is available as
-        ``self.absolute_path``.
-
-        .. versionadded:: 3.1
-        """
-        # os.path.abspath strips a trailing /.
-        # We must add it back to `root` so that we only match files
-        # in a directory named `root` instead of files starting with
-        # that prefix.
-        root = os.path.abspath(root)
-        if not root.endswith(os.path.sep):
-            # abspath always removes a trailing slash, except when
-            # root is '/'. This is an unusual case, but several projects
-            # have independently discovered this technique to disable
-            # Tornado's path validation and (hopefully) do their own,
-            # so we need to support it.
-            root += os.path.sep
-        # The trailing slash also needs to be temporarily added back
-        # the requested path so a request to root/ will match.
-        if not (absolute_path + os.path.sep).startswith(root):
-            raise HTTPError(403, "%s is not in root static directory",
-                            self.path)
-        if (os.path.isdir(absolute_path) and
-                self.default_filename is not None):
-            # need to look at the request.path here for when path is empty
-            # but there is some prefix to the path that was already
-            # trimmed by the routing
-            if not self.request.path.endswith("/"):
-                self.redirect(self.request.path + "/", permanent=True)
-                return
-            absolute_path = os.path.join(absolute_path, self.default_filename)
-        if not os.path.exists(absolute_path):
-            raise HTTPError(404)
-        if not os.path.isfile(absolute_path):
-            raise HTTPError(403, "%s is not a file", self.path)
-        return absolute_path
-
-    @classmethod
-    def get_content(cls, abspath, start=None, end=None):
-        """Retrieve the content of the requested resource which is located
-        at the given absolute path.
-
-        This class method may be overridden by subclasses.  Note that its
-        signature is different from other overridable class methods
-        (no ``settings`` argument); this is deliberate to ensure that
-        ``abspath`` is able to stand on its own as a cache key.
-
-        This method should either return a byte string or an iterator
-        of byte strings.  The latter is preferred for large files
-        as it helps reduce memory fragmentation.
-
-        .. versionadded:: 3.1
-        """
-        with open(abspath, "rb") as file:
-            if start is not None:
-                file.seek(start)
-            if end is not None:
-                remaining = end - (start or 0)
-            else:
-                remaining = None
-            while True:
-                chunk_size = 64 * 1024
-                if remaining is not None and remaining < chunk_size:
-                    chunk_size = remaining
-                chunk = file.read(chunk_size)
-                if chunk:
-                    if remaining is not None:
-                        remaining -= len(chunk)
-                    yield chunk
-                else:
-                    if remaining is not None:
-                        assert remaining == 0
-                    return
-
-    @classmethod
-    def get_content_version(cls, abspath):
-        """Returns a version string for the resource at the given path.
-
-        This class method may be overridden by subclasses.  The
-        default implementation is a hash of the file's contents.
-
-        .. versionadded:: 3.1
-        """
-        data = cls.get_content(abspath)
-        hasher = hashlib.md5()
-        if isinstance(data, bytes):
-            hasher.update(data)
-        else:
-            for chunk in data:
-                hasher.update(chunk)
-        return hasher.hexdigest()
-
-    def _stat(self):
-        if not hasattr(self, '_stat_result'):
-            self._stat_result = os.stat(self.absolute_path)
-        return self._stat_result
-
-    def get_content_size(self):
-        """Retrieve the total size of the resource at the given path.
-
-        This method may be overridden by subclasses.
-
-        .. versionadded:: 3.1
-
-        .. versionchanged:: 4.0
-           This method is now always called, instead of only when
-           partial results are requested.
-        """
-        stat_result = self._stat()
-        return stat_result[stat.ST_SIZE]
-
-    def get_modified_time(self):
-        """Returns the time that ``self.absolute_path`` was last modified.
-
-        May be overridden in subclasses.  Should return a `~datetime.datetime`
-        object or None.
-
-        .. versionadded:: 3.1
-        """
-        stat_result = self._stat()
-        modified = datetime.datetime.utcfromtimestamp(
-            stat_result[stat.ST_MTIME])
-        return modified
-
-    def get_content_type(self):
-        """Returns the ``Content-Type`` header to be used for this request.
-
-        .. versionadded:: 3.1
-        """
-        mime_type, encoding = mimetypes.guess_type(self.absolute_path)
-        # per RFC 6713, use the appropriate type for a gzip compressed file
-        if encoding == "gzip":
-            return "application/gzip"
-        # As of 2015-07-21 there is no bzip2 encoding defined at
-        # http://www.iana.org/assignments/media-types/media-types.xhtml
-        # So for that (and any other encoding), use octet-stream.
-        elif encoding is not None:
-            return "application/octet-stream"
-        elif mime_type is not None:
-            return mime_type
-        # if mime_type not detected, use application/octet-stream
-        else:
-            return "application/octet-stream"
-
-    def set_extra_headers(self, path):
-        """For subclass to add extra headers to the response"""
-        pass
-
-    def get_cache_time(self, path, modified, mime_type):
-        """Override to customize cache control behavior.
-
-        Return a positive number of seconds to make the result
-        cacheable for that amount of time or 0 to mark resource as
-        cacheable for an unspecified amount of time (subject to
-        browser heuristics).
-
-        By default returns cache expiry of 10 years for resources requested
-        with ``v`` argument.
-        """
-        return self.CACHE_MAX_AGE if "v" in self.request.arguments else 0
-
-    @classmethod
-    def make_static_url(cls, settings, path, include_version=True):
-        """Constructs a versioned url for the given path.
-
-        This method may be overridden in subclasses (but note that it
-        is a class method rather than an instance method).  Subclasses
-        are only required to implement the signature
-        ``make_static_url(cls, settings, path)``; other keyword
-        arguments may be passed through `~RequestHandler.static_url`
-        but are not standard.
-
-        ``settings`` is the `Application.settings` dictionary.  ``path``
-        is the static path being requested.  The url returned should be
-        relative to the current host.
-
-        ``include_version`` determines whether the generated URL should
-        include the query string containing the version hash of the
-        file corresponding to the given ``path``.
-
-        """
-        url = settings.get('static_url_prefix', '/static/') + path
-        if not include_version:
-            return url
-
-        version_hash = cls.get_version(settings, path)
-        if not version_hash:
-            return url
-
-        return '%s?v=%s' % (url, version_hash)
-
-    def parse_url_path(self, url_path):
-        """Converts a static URL path into a filesystem path.
-
-        ``url_path`` is the path component of the URL with
-        ``static_url_prefix`` removed.  The return value should be
-        filesystem path relative to ``static_path``.
-
-        This is the inverse of `make_static_url`.
-        """
-        if os.path.sep != "/":
-            url_path = url_path.replace("/", os.path.sep)
-        return url_path
-
-    @classmethod
-    def get_version(cls, settings, path):
-        """Generate the version string to be used in static URLs.
-
-        ``settings`` is the `Application.settings` dictionary and ``path``
-        is the relative location of the requested asset on the filesystem.
-        The returned value should be a string, or ``None`` if no version
-        could be determined.
-
-        .. versionchanged:: 3.1
-           This method was previously recommended for subclasses to override;
-           `get_content_version` is now preferred as it allows the base
-           class to handle caching of the result.
-        """
-        abs_path = cls.get_absolute_path(settings['static_path'], path)
-        return cls._get_cached_version(abs_path)
-
-    @classmethod
-    def _get_cached_version(cls, abs_path):
-        with cls._lock:
-            hashes = cls._static_hashes
-            if abs_path not in hashes:
-                try:
-                    hashes[abs_path] = cls.get_content_version(abs_path)
-                except Exception:
-                    gen_log.error("Could not open static file %r", abs_path)
-                    hashes[abs_path] = None
-            hsh = hashes.get(abs_path)
-            if hsh:
-                return hsh
-        return None
-
-
-class FallbackHandler(RequestHandler):
-    """A `RequestHandler` that wraps another HTTP server callback.
-
-    The fallback is a callable object that accepts an
-    `~.httputil.HTTPServerRequest`, such as an `Application` or
-    `tornado.wsgi.WSGIContainer`.  This is most useful to use both
-    Tornado ``RequestHandlers`` and WSGI in the same server.  Typical
-    usage::
-
-        wsgi_app = tornado.wsgi.WSGIContainer(
-            django.core.handlers.wsgi.WSGIHandler())
-        application = tornado.web.Application([
-            (r"/foo", FooHandler),
-            (r".*", FallbackHandler, dict(fallback=wsgi_app),
-        ])
-    """
-    def initialize(self, fallback):
-        self.fallback = fallback
-
-    def prepare(self):
-        self.fallback(self.request)
-        self._finished = True
-
-
-class OutputTransform(object):
-    """A transform modifies the result of an HTTP request (e.g., GZip encoding)
-
-    Applications are not expected to create their own OutputTransforms
-    or interact with them directly; the framework chooses which transforms
-    (if any) to apply.
-    """
-    def __init__(self, request):
-        pass
-
-    def transform_first_chunk(self, status_code, headers, chunk, finishing):
-        # type: (int, httputil.HTTPHeaders, bytes, bool) -> typing.Tuple[int, httputil.HTTPHeaders, bytes]
-        return status_code, headers, chunk
-
-    def transform_chunk(self, chunk, finishing):
-        return chunk
-
-
-class GZipContentEncoding(OutputTransform):
-    """Applies the gzip content encoding to the response.
-
-    See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11
-
-    .. versionchanged:: 4.0
-        Now compresses all mime types beginning with ``text/``, instead
-        of just a whitelist. (the whitelist is still used for certain
-        non-text mime types).
-    """
-    # Whitelist of compressible mime types (in addition to any types
-    # beginning with "text/").
-    CONTENT_TYPES = set(["application/javascript", "application/x-javascript",
-                         "application/xml", "application/atom+xml",
-                         "application/json", "application/xhtml+xml",
-                         "image/svg+xml"])
-    # Python's GzipFile defaults to level 9, while most other gzip
-    # tools (including gzip itself) default to 6, which is probably a
-    # better CPU/size tradeoff.
-    GZIP_LEVEL = 6
-    # Responses that are too short are unlikely to benefit from gzipping
-    # after considering the "Content-Encoding: gzip" header and the header
-    # inside the gzip encoding.
-    # Note that responses written in multiple chunks will be compressed
-    # regardless of size.
-    MIN_LENGTH = 1024
-
-    def __init__(self, request):
-        self._gzipping = "gzip" in request.headers.get("Accept-Encoding", "")
-
-    def _compressible_type(self, ctype):
-        return ctype.startswith('text/') or ctype in self.CONTENT_TYPES
-
-    def transform_first_chunk(self, status_code, headers, chunk, finishing):
-        # type: (int, httputil.HTTPHeaders, bytes, bool) -> typing.Tuple[int, httputil.HTTPHeaders, bytes]
-        # TODO: can/should this type be inherited from the superclass?
-        if 'Vary' in headers:
-            headers['Vary'] += ', Accept-Encoding'
-        else:
-            headers['Vary'] = 'Accept-Encoding'
-        if self._gzipping:
-            ctype = _unicode(headers.get("Content-Type", "")).split(";")[0]
-            self._gzipping = self._compressible_type(ctype) and \
-                (not finishing or len(chunk) >= self.MIN_LENGTH) and \
-                ("Content-Encoding" not in headers)
-        if self._gzipping:
-            headers["Content-Encoding"] = "gzip"
-            self._gzip_value = BytesIO()
-            self._gzip_file = gzip.GzipFile(mode="w", fileobj=self._gzip_value,
-                                            compresslevel=self.GZIP_LEVEL)
-            chunk = self.transform_chunk(chunk, finishing)
-            if "Content-Length" in headers:
-                # The original content length is no longer correct.
-                # If this is the last (and only) chunk, we can set the new
-                # content-length; otherwise we remove it and fall back to
-                # chunked encoding.
-                if finishing:
-                    headers["Content-Length"] = str(len(chunk))
-                else:
-                    del headers["Content-Length"]
-        return status_code, headers, chunk
-
-    def transform_chunk(self, chunk, finishing):
-        if self._gzipping:
-            self._gzip_file.write(chunk)
-            if finishing:
-                self._gzip_file.close()
-            else:
-                self._gzip_file.flush()
-            chunk = self._gzip_value.getvalue()
-            self._gzip_value.truncate(0)
-            self._gzip_value.seek(0)
-        return chunk
-
-
-def authenticated(method):
-    """Decorate methods with this to require that the user be logged in.
-
-    If the user is not logged in, they will be redirected to the configured
-    `login url `.
-
-    If you configure a login url with a query parameter, Tornado will
-    assume you know what you're doing and use it as-is.  If not, it
-    will add a `next` parameter so the login page knows where to send
-    you once you're logged in.
-    """
-    @functools.wraps(method)
-    def wrapper(self, *args, **kwargs):
-        if not self.current_user:
-            if self.request.method in ("GET", "HEAD"):
-                url = self.get_login_url()
-                if "?" not in url:
-                    if urlparse.urlsplit(url).scheme:
-                        # if login url is absolute, make next absolute too
-                        next_url = self.request.full_url()
-                    else:
-                        next_url = self.request.uri
-                    url += "?" + urlencode(dict(next=next_url))
-                self.redirect(url)
-                return
-            raise HTTPError(403)
-        return method(self, *args, **kwargs)
-    return wrapper
-
-
-class UIModule(object):
-    """A re-usable, modular UI unit on a page.
-
-    UI modules often execute additional queries, and they can include
-    additional CSS and JavaScript that will be included in the output
-    page, which is automatically inserted on page render.
-
-    Subclasses of UIModule must override the `render` method.
-    """
-    def __init__(self, handler):
-        self.handler = handler
-        self.request = handler.request
-        self.ui = handler.ui
-        self.locale = handler.locale
-
-    @property
-    def current_user(self):
-        return self.handler.current_user
-
-    def render(self, *args, **kwargs):
-        """Override in subclasses to return this module's output."""
-        raise NotImplementedError()
-
-    def embedded_javascript(self):
-        """Override to return a JavaScript string
-        to be embedded in the page."""
-        return None
-
-    def javascript_files(self):
-        """Override to return a list of JavaScript files needed by this module.
-
-        If the return values are relative paths, they will be passed to
-        `RequestHandler.static_url`; otherwise they will be used as-is.
-        """
-        return None
-
-    def embedded_css(self):
-        """Override to return a CSS string
-        that will be embedded in the page."""
-        return None
-
-    def css_files(self):
-        """Override to returns a list of CSS files required by this module.
-
-        If the return values are relative paths, they will be passed to
-        `RequestHandler.static_url`; otherwise they will be used as-is.
-        """
-        return None
-
-    def html_head(self):
-        """Override to return an HTML string that will be put in the 
-        element.
-        """
-        return None
-
-    def html_body(self):
-        """Override to return an HTML string that will be put at the end of
-        the  element.
-        """
-        return None
-
-    def render_string(self, path, **kwargs):
-        """Renders a template and returns it as a string."""
-        return self.handler.render_string(path, **kwargs)
-
-
-class _linkify(UIModule):
-    def render(self, text, **kwargs):
-        return escape.linkify(text, **kwargs)
-
-
-class _xsrf_form_html(UIModule):
-    def render(self):
-        return self.handler.xsrf_form_html()
-
-
-class TemplateModule(UIModule):
-    """UIModule that simply renders the given template.
-
-    {% module Template("foo.html") %} is similar to {% include "foo.html" %},
-    but the module version gets its own namespace (with kwargs passed to
-    Template()) instead of inheriting the outer template's namespace.
-
-    Templates rendered through this module also get access to UIModule's
-    automatic javascript/css features.  Simply call set_resources
-    inside the template and give it keyword arguments corresponding to
-    the methods on UIModule: {{ set_resources(js_files=static_url("my.js")) }}
-    Note that these resources are output once per template file, not once
-    per instantiation of the template, so they must not depend on
-    any arguments to the template.
-    """
-    def __init__(self, handler):
-        super(TemplateModule, self).__init__(handler)
-        # keep resources in both a list and a dict to preserve order
-        self._resource_list = []
-        self._resource_dict = {}
-
-    def render(self, path, **kwargs):
-        def set_resources(**kwargs):
-            if path not in self._resource_dict:
-                self._resource_list.append(kwargs)
-                self._resource_dict[path] = kwargs
-            else:
-                if self._resource_dict[path] != kwargs:
-                    raise ValueError("set_resources called with different "
-                                     "resources for the same template")
-            return ""
-        return self.render_string(path, set_resources=set_resources,
-                                  **kwargs)
-
-    def _get_resources(self, key):
-        return (r[key] for r in self._resource_list if key in r)
-
-    def embedded_javascript(self):
-        return "\n".join(self._get_resources("embedded_javascript"))
-
-    def javascript_files(self):
-        result = []
-        for f in self._get_resources("javascript_files"):
-            if isinstance(f, (unicode_type, bytes)):
-                result.append(f)
-            else:
-                result.extend(f)
-        return result
-
-    def embedded_css(self):
-        return "\n".join(self._get_resources("embedded_css"))
-
-    def css_files(self):
-        result = []
-        for f in self._get_resources("css_files"):
-            if isinstance(f, (unicode_type, bytes)):
-                result.append(f)
-            else:
-                result.extend(f)
-        return result
-
-    def html_head(self):
-        return "".join(self._get_resources("html_head"))
-
-    def html_body(self):
-        return "".join(self._get_resources("html_body"))
-
-
-class _UIModuleNamespace(object):
-    """Lazy namespace which creates UIModule proxies bound to a handler."""
-    def __init__(self, handler, ui_modules):
-        self.handler = handler
-        self.ui_modules = ui_modules
-
-    def __getitem__(self, key):
-        return self.handler._ui_module(key, self.ui_modules[key])
-
-    def __getattr__(self, key):
-        try:
-            return self[key]
-        except KeyError as e:
-            raise AttributeError(str(e))
-
-
-if hasattr(hmac, 'compare_digest'):  # python 3.3
-    _time_independent_equals = hmac.compare_digest
-else:
-    def _time_independent_equals(a, b):
-        if len(a) != len(b):
-            return False
-        result = 0
-        if isinstance(a[0], int):  # python3 byte strings
-            for x, y in zip(a, b):
-                result |= x ^ y
-        else:  # python2
-            for x, y in zip(a, b):
-                result |= ord(x) ^ ord(y)
-        return result == 0
-
-
-def create_signed_value(secret, name, value, version=None, clock=None,
-                        key_version=None):
-    if version is None:
-        version = DEFAULT_SIGNED_VALUE_VERSION
-    if clock is None:
-        clock = time.time
-
-    timestamp = utf8(str(int(clock())))
-    value = base64.b64encode(utf8(value))
-    if version == 1:
-        signature = _create_signature_v1(secret, name, value, timestamp)
-        value = b"|".join([value, timestamp, signature])
-        return value
-    elif version == 2:
-        # The v2 format consists of a version number and a series of
-        # length-prefixed fields "%d:%s", the last of which is a
-        # signature, all separated by pipes.  All numbers are in
-        # decimal format with no leading zeros.  The signature is an
-        # HMAC-SHA256 of the whole string up to that point, including
-        # the final pipe.
-        #
-        # The fields are:
-        # - format version (i.e. 2; no length prefix)
-        # - key version (integer, default is 0)
-        # - timestamp (integer seconds since epoch)
-        # - name (not encoded; assumed to be ~alphanumeric)
-        # - value (base64-encoded)
-        # - signature (hex-encoded; no length prefix)
-        def format_field(s):
-            return utf8("%d:" % len(s)) + utf8(s)
-        to_sign = b"|".join([
-            b"2",
-            format_field(str(key_version or 0)),
-            format_field(timestamp),
-            format_field(name),
-            format_field(value),
-            b''])
-
-        if isinstance(secret, dict):
-            assert key_version is not None, 'Key version must be set when sign key dict is used'
-            assert version >= 2, 'Version must be at least 2 for key version support'
-            secret = secret[key_version]
-
-        signature = _create_signature_v2(secret, to_sign)
-        return to_sign + signature
-    else:
-        raise ValueError("Unsupported version %d" % version)
-
-
-# A leading version number in decimal
-# with no leading zeros, followed by a pipe.
-_signed_value_version_re = re.compile(br"^([1-9][0-9]*)\|(.*)$")
-
-
-def _get_version(value):
-    # Figures out what version value is.  Version 1 did not include an
-    # explicit version field and started with arbitrary base64 data,
-    # which makes this tricky.
-    m = _signed_value_version_re.match(value)
-    if m is None:
-        version = 1
-    else:
-        try:
-            version = int(m.group(1))
-            if version > 999:
-                # Certain payloads from the version-less v1 format may
-                # be parsed as valid integers.  Due to base64 padding
-                # restrictions, this can only happen for numbers whose
-                # length is a multiple of 4, so we can treat all
-                # numbers up to 999 as versions, and for the rest we
-                # fall back to v1 format.
-                version = 1
-        except ValueError:
-            version = 1
-    return version
-
-
-def decode_signed_value(secret, name, value, max_age_days=31,
-                        clock=None, min_version=None):
-    if clock is None:
-        clock = time.time
-    if min_version is None:
-        min_version = DEFAULT_SIGNED_VALUE_MIN_VERSION
-    if min_version > 2:
-        raise ValueError("Unsupported min_version %d" % min_version)
-    if not value:
-        return None
-
-    value = utf8(value)
-    version = _get_version(value)
-
-    if version < min_version:
-        return None
-    if version == 1:
-        return _decode_signed_value_v1(secret, name, value,
-                                       max_age_days, clock)
-    elif version == 2:
-        return _decode_signed_value_v2(secret, name, value,
-                                       max_age_days, clock)
-    else:
-        return None
-
-
-def _decode_signed_value_v1(secret, name, value, max_age_days, clock):
-    parts = utf8(value).split(b"|")
-    if len(parts) != 3:
-        return None
-    signature = _create_signature_v1(secret, name, parts[0], parts[1])
-    if not _time_independent_equals(parts[2], signature):
-        gen_log.warning("Invalid cookie signature %r", value)
-        return None
-    timestamp = int(parts[1])
-    if timestamp < clock() - max_age_days * 86400:
-        gen_log.warning("Expired cookie %r", value)
-        return None
-    if timestamp > clock() + 31 * 86400:
-        # _cookie_signature does not hash a delimiter between the
-        # parts of the cookie, so an attacker could transfer trailing
-        # digits from the payload to the timestamp without altering the
-        # signature.  For backwards compatibility, sanity-check timestamp
-        # here instead of modifying _cookie_signature.
-        gen_log.warning("Cookie timestamp in future; possible tampering %r",
-                        value)
-        return None
-    if parts[1].startswith(b"0"):
-        gen_log.warning("Tampered cookie %r", value)
-        return None
-    try:
-        return base64.b64decode(parts[0])
-    except Exception:
-        return None
-
-
-def _decode_fields_v2(value):
-    def _consume_field(s):
-        length, _, rest = s.partition(b':')
-        n = int(length)
-        field_value = rest[:n]
-        # In python 3, indexing bytes returns small integers; we must
-        # use a slice to get a byte string as in python 2.
-        if rest[n:n + 1] != b'|':
-            raise ValueError("malformed v2 signed value field")
-        rest = rest[n + 1:]
-        return field_value, rest
-
-    rest = value[2:]  # remove version number
-    key_version, rest = _consume_field(rest)
-    timestamp, rest = _consume_field(rest)
-    name_field, rest = _consume_field(rest)
-    value_field, passed_sig = _consume_field(rest)
-    return int(key_version), timestamp, name_field, value_field, passed_sig
-
-
-def _decode_signed_value_v2(secret, name, value, max_age_days, clock):
-    try:
-        key_version, timestamp, name_field, value_field, passed_sig = _decode_fields_v2(value)
-    except ValueError:
-        return None
-    signed_string = value[:-len(passed_sig)]
-
-    if isinstance(secret, dict):
-        try:
-            secret = secret[key_version]
-        except KeyError:
-            return None
-
-    expected_sig = _create_signature_v2(secret, signed_string)
-    if not _time_independent_equals(passed_sig, expected_sig):
-        return None
-    if name_field != utf8(name):
-        return None
-    timestamp = int(timestamp)
-    if timestamp < clock() - max_age_days * 86400:
-        # The signature has expired.
-        return None
-    try:
-        return base64.b64decode(value_field)
-    except Exception:
-        return None
-
-
-def get_signature_key_version(value):
-    value = utf8(value)
-    version = _get_version(value)
-    if version < 2:
-        return None
-    try:
-        key_version, _, _, _, _ = _decode_fields_v2(value)
-    except ValueError:
-        return None
-
-    return key_version
-
-
-def _create_signature_v1(secret, *parts):
-    hash = hmac.new(utf8(secret), digestmod=hashlib.sha1)
-    for part in parts:
-        hash.update(utf8(part))
-    return utf8(hash.hexdigest())
-
-
-def _create_signature_v2(secret, s):
-    hash = hmac.new(utf8(secret), digestmod=hashlib.sha256)
-    hash.update(utf8(s))
-    return utf8(hash.hexdigest())
-
-
-def is_absolute(path):
-    return any(path.startswith(x) for x in ["/", "http:", "https:"])
diff --git a/salt/ext/tornado/websocket.py b/salt/ext/tornado/websocket.py
deleted file mode 100644
index 47cb9770b25..00000000000
--- a/salt/ext/tornado/websocket.py
+++ /dev/null
@@ -1,1245 +0,0 @@
-"""Implementation of the WebSocket protocol.
-
-`WebSockets `_ allow for bidirectional
-communication between the browser and server.
-
-WebSockets are supported in the current versions of all major browsers,
-although older versions that do not support WebSockets are still in use
-(refer to http://caniuse.com/websockets for details).
-
-This module implements the final version of the WebSocket protocol as
-defined in `RFC 6455 `_.  Certain
-browser versions (notably Safari 5.x) implemented an earlier draft of
-the protocol (known as "draft 76") and are not compatible with this module.
-
-.. versionchanged:: 4.0
-   Removed support for the draft 76 protocol version.
-"""
-# pylint: skip-file
-
-from __future__ import absolute_import, division, print_function
-# Author: Jacob Kristhammar, 2010
-
-import base64
-import collections
-import hashlib
-import os
-import struct
-import salt.ext.tornado.escape as tornado_escape
-import salt.ext.tornado.web as tornado_web
-import zlib
-
-from salt.ext.tornado.concurrent import TracebackFuture
-from salt.ext.tornado.escape import utf8, native_str, to_unicode
-from salt.ext.tornado import gen, httpclient, httputil
-from salt.ext.tornado.ioloop import IOLoop, PeriodicCallback
-from salt.ext.tornado.iostream import StreamClosedError
-from salt.ext.tornado.log import gen_log, app_log
-from salt.ext.tornado import simple_httpclient
-from salt.ext.tornado.tcpclient import TCPClient
-from salt.ext.tornado.util import _websocket_mask, PY3
-
-if PY3:
-    from urllib.parse import urlparse  # py2
-    xrange = range
-else:
-    from urlparse import urlparse  # py3
-
-
-class WebSocketError(Exception):
-    pass
-
-
-class WebSocketClosedError(WebSocketError):
-    """Raised by operations on a closed connection.
-
-    .. versionadded:: 3.2
-    """
-    pass
-
-
-class WebSocketHandler(tornado_web.RequestHandler):
-    """Subclass this class to create a basic WebSocket handler.
-
-    Override `on_message` to handle incoming messages, and use
-    `write_message` to send messages to the client. You can also
-    override `open` and `on_close` to handle opened and closed
-    connections.
-
-    Custom upgrade response headers can be sent by overriding
-    `~tornado.web.RequestHandler.set_default_headers` or
-    `~tornado.web.RequestHandler.prepare`.
-
-    See http://dev.w3.org/html5/websockets/ for details on the
-    JavaScript interface.  The protocol is specified at
-    http://tools.ietf.org/html/rfc6455.
-
-    Here is an example WebSocket handler that echos back all received messages
-    back to the client:
-
-    .. testcode::
-
-      class EchoWebSocket(tornado.websocket.WebSocketHandler):
-          def open(self):
-              print("WebSocket opened")
-
-          def on_message(self, message):
-              self.write_message(u"You said: " + message)
-
-          def on_close(self):
-              print("WebSocket closed")
-
-    .. testoutput::
-       :hide:
-
-    WebSockets are not standard HTTP connections. The "handshake" is
-    HTTP, but after the handshake, the protocol is
-    message-based. Consequently, most of the Tornado HTTP facilities
-    are not available in handlers of this type. The only communication
-    methods available to you are `write_message()`, `ping()`, and
-    `close()`. Likewise, your request handler class should implement
-    `open()` method rather than ``get()`` or ``post()``.
-
-    If you map the handler above to ``/websocket`` in your application, you can
-    invoke it in JavaScript with::
-
-      var ws = new WebSocket("ws://localhost:8888/websocket");
-      ws.onopen = function() {
-         ws.send("Hello, world");
-      };
-      ws.onmessage = function (evt) {
-         alert(evt.data);
-      };
-
-    This script pops up an alert box that says "You said: Hello, world".
-
-    Web browsers allow any site to open a websocket connection to any other,
-    instead of using the same-origin policy that governs other network
-    access from javascript.  This can be surprising and is a potential
-    security hole, so since Tornado 4.0 `WebSocketHandler` requires
-    applications that wish to receive cross-origin websockets to opt in
-    by overriding the `~WebSocketHandler.check_origin` method (see that
-    method's docs for details).  Failure to do so is the most likely
-    cause of 403 errors when making a websocket connection.
-
-    When using a secure websocket connection (``wss://``) with a self-signed
-    certificate, the connection from a browser may fail because it wants
-    to show the "accept this certificate" dialog but has nowhere to show it.
-    You must first visit a regular HTML page using the same certificate
-    to accept it before the websocket connection will succeed.
-
-    If the application setting ``websocket_ping_interval`` has a non-zero
-    value, a ping will be sent periodically, and the connection will be
-    closed if a response is not received before the ``websocket_ping_timeout``.
-
-    Messages larger than the ``websocket_max_message_size`` application setting
-    (default 10MiB) will not be accepted.
-
-    .. versionchanged:: 4.5
-       Added ``websocket_ping_interval``, ``websocket_ping_timeout``, and
-       ``websocket_max_message_size``.
-    """
-    def __init__(self, application, request, **kwargs):
-        super(WebSocketHandler, self).__init__(application, request, **kwargs)
-        self.ws_connection = None
-        self.close_code = None
-        self.close_reason = None
-        self.stream = None
-        self._on_close_called = False
-
-    @tornado_web.asynchronous
-    def get(self, *args, **kwargs):
-        self.open_args = args
-        self.open_kwargs = kwargs
-
-        # Upgrade header should be present and should be equal to WebSocket
-        if self.request.headers.get("Upgrade", "").lower() != 'websocket':
-            self.set_status(400)
-            log_msg = "Can \"Upgrade\" only to \"WebSocket\"."
-            self.finish(log_msg)
-            gen_log.debug(log_msg)
-            return
-
-        # Connection header should be upgrade.
-        # Some proxy servers/load balancers
-        # might mess with it.
-        headers = self.request.headers
-        connection = map(lambda s: s.strip().lower(),
-                         headers.get("Connection", "").split(","))
-        if 'upgrade' not in connection:
-            self.set_status(400)
-            log_msg = "\"Connection\" must be \"Upgrade\"."
-            self.finish(log_msg)
-            gen_log.debug(log_msg)
-            return
-
-        # Handle WebSocket Origin naming convention differences
-        # The difference between version 8 and 13 is that in 8 the
-        # client sends a "Sec-Websocket-Origin" header and in 13 it's
-        # simply "Origin".
-        if "Origin" in self.request.headers:
-            origin = self.request.headers.get("Origin")
-        else:
-            origin = self.request.headers.get("Sec-Websocket-Origin", None)
-
-        # If there was an origin header, check to make sure it matches
-        # according to check_origin. When the origin is None, we assume it
-        # did not come from a browser and that it can be passed on.
-        if origin is not None and not self.check_origin(origin):
-            self.set_status(403)
-            log_msg = "Cross origin websockets not allowed"
-            self.finish(log_msg)
-            gen_log.debug(log_msg)
-            return
-
-        self.ws_connection = self.get_websocket_protocol()
-        if self.ws_connection:
-            self.ws_connection.accept_connection()
-        else:
-            self.set_status(426, "Upgrade Required")
-            self.set_header("Sec-WebSocket-Version", "7, 8, 13")
-            self.finish()
-
-    stream = None
-
-    @property
-    def ping_interval(self):
-        """The interval for websocket keep-alive pings.
-
-        Set websocket_ping_interval = 0 to disable pings.
-        """
-        return self.settings.get('websocket_ping_interval', None)
-
-    @property
-    def ping_timeout(self):
-        """If no ping is received in this many seconds,
-        close the websocket connection (VPNs, etc. can fail to cleanly close ws connections).
-        Default is max of 3 pings or 30 seconds.
-        """
-        return self.settings.get('websocket_ping_timeout', None)
-
-    @property
-    def max_message_size(self):
-        """Maximum allowed message size.
-
-        If the remote peer sends a message larger than this, the connection
-        will be closed.
-
-        Default is 10MiB.
-        """
-        return self.settings.get('websocket_max_message_size', None)
-
-    def write_message(self, message, binary=False):
-        """Sends the given message to the client of this Web Socket.
-
-        The message may be either a string or a dict (which will be
-        encoded as json).  If the ``binary`` argument is false, the
-        message will be sent as utf8; in binary mode any byte string
-        is allowed.
-
-        If the connection is already closed, raises `WebSocketClosedError`.
-
-        .. versionchanged:: 3.2
-           `WebSocketClosedError` was added (previously a closed connection
-           would raise an `AttributeError`)
-
-        .. versionchanged:: 4.3
-           Returns a `.Future` which can be used for flow control.
-        """
-        if self.ws_connection is None:
-            raise WebSocketClosedError()
-        if isinstance(message, dict):
-            message = tornado_escape.json_encode(message)
-        return self.ws_connection.write_message(message, binary=binary)
-
-    def select_subprotocol(self, subprotocols):
-        """Invoked when a new WebSocket requests specific subprotocols.
-
-        ``subprotocols`` is a list of strings identifying the
-        subprotocols proposed by the client.  This method may be
-        overridden to return one of those strings to select it, or
-        ``None`` to not select a subprotocol.  Failure to select a
-        subprotocol does not automatically abort the connection,
-        although clients may close the connection if none of their
-        proposed subprotocols was selected.
-        """
-        return None
-
-    def get_compression_options(self):
-        """Override to return compression options for the connection.
-
-        If this method returns None (the default), compression will
-        be disabled.  If it returns a dict (even an empty one), it
-        will be enabled.  The contents of the dict may be used to
-        control the following compression options:
-
-        ``compression_level`` specifies the compression level.
-
-        ``mem_level`` specifies the amount of memory used for the internal compression state.
-
-         These parameters are documented in details here:
-         https://docs.python.org/3.6/library/zlib.html#zlib.compressobj
-
-        .. versionadded:: 4.1
-
-        .. versionchanged:: 4.5
-
-           Added ``compression_level`` and ``mem_level``.
-        """
-        # TODO: Add wbits option.
-        return None
-
-    def open(self, *args, **kwargs):
-        """Invoked when a new WebSocket is opened.
-
-        The arguments to `open` are extracted from the `tornado.web.URLSpec`
-        regular expression, just like the arguments to
-        `tornado.web.RequestHandler.get`.
-        """
-        pass
-
-    def on_message(self, message):
-        """Handle incoming messages on the WebSocket
-
-        This method must be overridden.
-
-        .. versionchanged:: 4.5
-
-           ``on_message`` can be a coroutine.
-        """
-        raise NotImplementedError
-
-    def ping(self, data):
-        """Send ping frame to the remote end."""
-        if self.ws_connection is None:
-            raise WebSocketClosedError()
-        self.ws_connection.write_ping(data)
-
-    def on_pong(self, data):
-        """Invoked when the response to a ping frame is received."""
-        pass
-
-    def on_ping(self, data):
-        """Invoked when the a ping frame is received."""
-        pass
-
-    def on_close(self):
-        """Invoked when the WebSocket is closed.
-
-        If the connection was closed cleanly and a status code or reason
-        phrase was supplied, these values will be available as the attributes
-        ``self.close_code`` and ``self.close_reason``.
-
-        .. versionchanged:: 4.0
-
-           Added ``close_code`` and ``close_reason`` attributes.
-        """
-        pass
-
-    def close(self, code=None, reason=None):
-        """Closes this Web Socket.
-
-        Once the close handshake is successful the socket will be closed.
-
-        ``code`` may be a numeric status code, taken from the values
-        defined in `RFC 6455 section 7.4.1
-        `_.
-        ``reason`` may be a textual message about why the connection is
-        closing.  These values are made available to the client, but are
-        not otherwise interpreted by the websocket protocol.
-
-        .. versionchanged:: 4.0
-
-           Added the ``code`` and ``reason`` arguments.
-        """
-        if self.ws_connection:
-            self.ws_connection.close(code, reason)
-            self.ws_connection = None
-
-    def check_origin(self, origin):
-        """Override to enable support for allowing alternate origins.
-
-        The ``origin`` argument is the value of the ``Origin`` HTTP
-        header, the url responsible for initiating this request.  This
-        method is not called for clients that do not send this header;
-        such requests are always allowed (because all browsers that
-        implement WebSockets support this header, and non-browser
-        clients do not have the same cross-site security concerns).
-
-        Should return True to accept the request or False to reject it.
-        By default, rejects all requests with an origin on a host other
-        than this one.
-
-        This is a security protection against cross site scripting attacks on
-        browsers, since WebSockets are allowed to bypass the usual same-origin
-        policies and don't use CORS headers.
-
-        .. warning::
-
-           This is an important security measure; don't disable it
-           without understanding the security implications. In
-           particular, if your authentication is cookie-based, you
-           must either restrict the origins allowed by
-           ``check_origin()`` or implement your own XSRF-like
-           protection for websocket connections. See `these
-           `_
-           `articles
-           `_
-           for more.
-
-        To accept all cross-origin traffic (which was the default prior to
-        Tornado 4.0), simply override this method to always return true::
-
-            def check_origin(self, origin):
-                return True
-
-        To allow connections from any subdomain of your site, you might
-        do something like::
-
-            def check_origin(self, origin):
-                parsed_origin = urllib.parse.urlparse(origin)
-                return parsed_origin.netloc.endswith(".mydomain.com")
-
-        .. versionadded:: 4.0
-
-        """
-        parsed_origin = urlparse(origin)
-        origin = parsed_origin.netloc
-        origin = origin.lower()
-
-        host = self.request.headers.get("Host")
-
-        # Check to see that origin matches host directly, including ports
-        return origin == host
-
-    def set_nodelay(self, value):
-        """Set the no-delay flag for this stream.
-
-        By default, small messages may be delayed and/or combined to minimize
-        the number of packets sent.  This can sometimes cause 200-500ms delays
-        due to the interaction between Nagle's algorithm and TCP delayed
-        ACKs.  To reduce this delay (at the expense of possibly increasing
-        bandwidth usage), call ``self.set_nodelay(True)`` once the websocket
-        connection is established.
-
-        See `.BaseIOStream.set_nodelay` for additional details.
-
-        .. versionadded:: 3.1
-        """
-        self.stream.set_nodelay(value)
-
-    def on_connection_close(self):
-        if self.ws_connection:
-            self.ws_connection.on_connection_close()
-            self.ws_connection = None
-        if not self._on_close_called:
-            self._on_close_called = True
-            self.on_close()
-            self._break_cycles()
-
-    def _break_cycles(self):
-        # WebSocketHandlers call finish() early, but we don't want to
-        # break up reference cycles (which makes it impossible to call
-        # self.render_string) until after we've really closed the
-        # connection (if it was established in the first place,
-        # indicated by status code 101).
-        if self.get_status() != 101 or self._on_close_called:
-            super(WebSocketHandler, self)._break_cycles()
-
-    def send_error(self, *args, **kwargs):
-        if self.stream is None:
-            super(WebSocketHandler, self).send_error(*args, **kwargs)
-        else:
-            # If we get an uncaught exception during the handshake,
-            # we have no choice but to abruptly close the connection.
-            # TODO: for uncaught exceptions after the handshake,
-            # we can close the connection more gracefully.
-            self.stream.close()
-
-    def get_websocket_protocol(self):
-        websocket_version = self.request.headers.get("Sec-WebSocket-Version")
-        if websocket_version in ("7", "8", "13"):
-            return WebSocketProtocol13(
-                self, compression_options=self.get_compression_options())
-
-    def _attach_stream(self):
-        self.stream = self.request.connection.detach()
-        self.stream.set_close_callback(self.on_connection_close)
-        # disable non-WS methods
-        for method in ["write", "redirect", "set_header", "set_cookie",
-                       "set_status", "flush", "finish"]:
-            setattr(self, method, _raise_not_supported_for_websockets)
-
-
-def _raise_not_supported_for_websockets(*args, **kwargs):
-    raise RuntimeError("Method not supported for Web Sockets")
-
-
-class WebSocketProtocol(object):
-    """Base class for WebSocket protocol versions.
-    """
-    def __init__(self, handler):
-        self.handler = handler
-        self.request = handler.request
-        self.stream = handler.stream
-        self.client_terminated = False
-        self.server_terminated = False
-
-    def _run_callback(self, callback, *args, **kwargs):
-        """Runs the given callback with exception handling.
-
-        If the callback is a coroutine, returns its Future. On error, aborts the
-        websocket connection and returns None.
-        """
-        try:
-            result = callback(*args, **kwargs)
-        except Exception:
-            app_log.error("Uncaught exception in %s",
-                          getattr(self.request, 'path', None), exc_info=True)
-            self._abort()
-        else:
-            if result is not None:
-                result = gen.convert_yielded(result)
-                self.stream.io_loop.add_future(result, lambda f: f.result())
-            return result
-
-    def on_connection_close(self):
-        self._abort()
-
-    def _abort(self):
-        """Instantly aborts the WebSocket connection by closing the socket"""
-        self.client_terminated = True
-        self.server_terminated = True
-        self.stream.close()  # forcibly tear down the connection
-        self.close()  # let the subclass cleanup
-
-
-class _PerMessageDeflateCompressor(object):
-    def __init__(self, persistent, max_wbits, compression_options=None):
-        if max_wbits is None:
-            max_wbits = zlib.MAX_WBITS
-        # There is no symbolic constant for the minimum wbits value.
-        if not (8 <= max_wbits <= zlib.MAX_WBITS):
-            raise ValueError("Invalid max_wbits value %r; allowed range 8-%d",
-                             max_wbits, zlib.MAX_WBITS)
-        self._max_wbits = max_wbits
-
-        if compression_options is None or 'compression_level' not in compression_options:
-            self._compression_level = tornado_web.GZipContentEncoding.GZIP_LEVEL
-        else:
-            self._compression_level = compression_options['compression_level']
-
-        if compression_options is None or 'mem_level' not in compression_options:
-            self._mem_level = 8
-        else:
-            self._mem_level = compression_options['mem_level']
-
-        if persistent:
-            self._compressor = self._create_compressor()
-        else:
-            self._compressor = None
-
-    def _create_compressor(self):
-        return zlib.compressobj(self._compression_level, zlib.DEFLATED, -self._max_wbits, self._mem_level)
-
-    def compress(self, data):
-        compressor = self._compressor or self._create_compressor()
-        data = (compressor.compress(data) +
-                compressor.flush(zlib.Z_SYNC_FLUSH))
-        assert data.endswith(b'\x00\x00\xff\xff')
-        return data[:-4]
-
-
-class _PerMessageDeflateDecompressor(object):
-    def __init__(self, persistent, max_wbits, compression_options=None):
-        if max_wbits is None:
-            max_wbits = zlib.MAX_WBITS
-        if not (8 <= max_wbits <= zlib.MAX_WBITS):
-            raise ValueError("Invalid max_wbits value %r; allowed range 8-%d",
-                             max_wbits, zlib.MAX_WBITS)
-        self._max_wbits = max_wbits
-        if persistent:
-            self._decompressor = self._create_decompressor()
-        else:
-            self._decompressor = None
-
-    def _create_decompressor(self):
-        return zlib.decompressobj(-self._max_wbits)
-
-    def decompress(self, data):
-        decompressor = self._decompressor or self._create_decompressor()
-        return decompressor.decompress(data + b'\x00\x00\xff\xff')
-
-
-class WebSocketProtocol13(WebSocketProtocol):
-    """Implementation of the WebSocket protocol from RFC 6455.
-
-    This class supports versions 7 and 8 of the protocol in addition to the
-    final version 13.
-    """
-    # Bit masks for the first byte of a frame.
-    FIN = 0x80
-    RSV1 = 0x40
-    RSV2 = 0x20
-    RSV3 = 0x10
-    RSV_MASK = RSV1 | RSV2 | RSV3
-    OPCODE_MASK = 0x0f
-
-    def __init__(self, handler, mask_outgoing=False,
-                 compression_options=None):
-        WebSocketProtocol.__init__(self, handler)
-        self.mask_outgoing = mask_outgoing
-        self._final_frame = False
-        self._frame_opcode = None
-        self._masked_frame = None
-        self._frame_mask = None
-        self._frame_length = None
-        self._fragmented_message_buffer = None
-        self._fragmented_message_opcode = None
-        self._waiting = None
-        self._compression_options = compression_options
-        self._decompressor = None
-        self._compressor = None
-        self._frame_compressed = None
-        # The total uncompressed size of all messages received or sent.
-        # Unicode messages are encoded to utf8.
-        # Only for testing; subject to change.
-        self._message_bytes_in = 0
-        self._message_bytes_out = 0
-        # The total size of all packets received or sent.  Includes
-        # the effect of compression, frame overhead, and control frames.
-        self._wire_bytes_in = 0
-        self._wire_bytes_out = 0
-        self.ping_callback = None
-        self.last_ping = 0
-        self.last_pong = 0
-
-    def accept_connection(self):
-        try:
-            self._handle_websocket_headers()
-        except ValueError:
-            self.handler.set_status(400)
-            log_msg = "Missing/Invalid WebSocket headers"
-            self.handler.finish(log_msg)
-            gen_log.debug(log_msg)
-            return
-
-        try:
-            self._accept_connection()
-        except ValueError:
-            gen_log.debug("Malformed WebSocket request received",
-                          exc_info=True)
-            self._abort()
-            return
-
-    def _handle_websocket_headers(self):
-        """Verifies all invariant- and required headers
-
-        If a header is missing or have an incorrect value ValueError will be
-        raised
-        """
-        fields = ("Host", "Sec-Websocket-Key", "Sec-Websocket-Version")
-        if not all(map(lambda f: self.request.headers.get(f), fields)):
-            raise ValueError("Missing/Invalid WebSocket headers")
-
-    @staticmethod
-    def compute_accept_value(key):
-        """Computes the value for the Sec-WebSocket-Accept header,
-        given the value for Sec-WebSocket-Key.
-        """
-        sha1 = hashlib.sha1()
-        sha1.update(utf8(key))
-        sha1.update(b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11")  # Magic value
-        return native_str(base64.b64encode(sha1.digest()))
-
-    def _challenge_response(self):
-        return WebSocketProtocol13.compute_accept_value(
-            self.request.headers.get("Sec-Websocket-Key"))
-
-    def _accept_connection(self):
-        subprotocols = self.request.headers.get("Sec-WebSocket-Protocol", '')
-        subprotocols = [s.strip() for s in subprotocols.split(',')]
-        if subprotocols:
-            selected = self.handler.select_subprotocol(subprotocols)
-            if selected:
-                assert selected in subprotocols
-                self.handler.set_header("Sec-WebSocket-Protocol", selected)
-
-        extensions = self._parse_extensions_header(self.request.headers)
-        for ext in extensions:
-            if (ext[0] == 'permessage-deflate' and
-                    self._compression_options is not None):
-                # TODO: negotiate parameters if compression_options
-                # specifies limits.
-                self._create_compressors('server', ext[1], self._compression_options)
-                if ('client_max_window_bits' in ext[1] and
-                        ext[1]['client_max_window_bits'] is None):
-                    # Don't echo an offered client_max_window_bits
-                    # parameter with no value.
-                    del ext[1]['client_max_window_bits']
-                self.handler.set_header("Sec-WebSocket-Extensions",
-                                        httputil._encode_header(
-                                            'permessage-deflate', ext[1]))
-                break
-
-        self.handler.clear_header("Content-Type")
-        self.handler.set_status(101)
-        self.handler.set_header("Upgrade", "websocket")
-        self.handler.set_header("Connection", "Upgrade")
-        self.handler.set_header("Sec-WebSocket-Accept", self._challenge_response())
-        self.handler.finish()
-
-        self.handler._attach_stream()
-        self.stream = self.handler.stream
-
-        self.start_pinging()
-        self._run_callback(self.handler.open, *self.handler.open_args,
-                           **self.handler.open_kwargs)
-        self._receive_frame()
-
-    def _parse_extensions_header(self, headers):
-        extensions = headers.get("Sec-WebSocket-Extensions", '')
-        if extensions:
-            return [httputil._parse_header(e.strip())
-                    for e in extensions.split(',')]
-        return []
-
-    def _process_server_headers(self, key, headers):
-        """Process the headers sent by the server to this client connection.
-
-        'key' is the websocket handshake challenge/response key.
-        """
-        assert headers['Upgrade'].lower() == 'websocket'
-        assert headers['Connection'].lower() == 'upgrade'
-        accept = self.compute_accept_value(key)
-        assert headers['Sec-Websocket-Accept'] == accept
-
-        extensions = self._parse_extensions_header(headers)
-        for ext in extensions:
-            if (ext[0] == 'permessage-deflate' and
-                    self._compression_options is not None):
-                self._create_compressors('client', ext[1])
-            else:
-                raise ValueError("unsupported extension %r", ext)
-
-    def _get_compressor_options(self, side, agreed_parameters, compression_options=None):
-        """Converts a websocket agreed_parameters set to keyword arguments
-        for our compressor objects.
-        """
-        options = dict(
-            persistent=(side + '_no_context_takeover') not in agreed_parameters)
-        wbits_header = agreed_parameters.get(side + '_max_window_bits', None)
-        if wbits_header is None:
-            options['max_wbits'] = zlib.MAX_WBITS
-        else:
-            options['max_wbits'] = int(wbits_header)
-        options['compression_options'] = compression_options
-        return options
-
-    def _create_compressors(self, side, agreed_parameters, compression_options=None):
-        # TODO: handle invalid parameters gracefully
-        allowed_keys = set(['server_no_context_takeover',
-                            'client_no_context_takeover',
-                            'server_max_window_bits',
-                            'client_max_window_bits'])
-        for key in agreed_parameters:
-            if key not in allowed_keys:
-                raise ValueError("unsupported compression parameter %r" % key)
-        other_side = 'client' if (side == 'server') else 'server'
-        self._compressor = _PerMessageDeflateCompressor(
-            **self._get_compressor_options(side, agreed_parameters, compression_options))
-        self._decompressor = _PerMessageDeflateDecompressor(
-            **self._get_compressor_options(other_side, agreed_parameters, compression_options))
-
-    def _write_frame(self, fin, opcode, data, flags=0):
-        if fin:
-            finbit = self.FIN
-        else:
-            finbit = 0
-        frame = struct.pack("B", finbit | opcode | flags)
-        l = len(data)
-        if self.mask_outgoing:
-            mask_bit = 0x80
-        else:
-            mask_bit = 0
-        if l < 126:
-            frame += struct.pack("B", l | mask_bit)
-        elif l <= 0xFFFF:
-            frame += struct.pack("!BH", 126 | mask_bit, l)
-        else:
-            frame += struct.pack("!BQ", 127 | mask_bit, l)
-        if self.mask_outgoing:
-            mask = os.urandom(4)
-            data = mask + _websocket_mask(mask, data)
-        frame += data
-        self._wire_bytes_out += len(frame)
-        try:
-            return self.stream.write(frame)
-        except StreamClosedError:
-            self._abort()
-
-    def write_message(self, message, binary=False):
-        """Sends the given message to the client of this Web Socket."""
-        if binary:
-            opcode = 0x2
-        else:
-            opcode = 0x1
-        message = tornado_escape.utf8(message)
-        assert isinstance(message, bytes)
-        self._message_bytes_out += len(message)
-        flags = 0
-        if self._compressor:
-            message = self._compressor.compress(message)
-            flags |= self.RSV1
-        return self._write_frame(True, opcode, message, flags=flags)
-
-    def write_ping(self, data):
-        """Send ping frame."""
-        assert isinstance(data, bytes)
-        self._write_frame(True, 0x9, data)
-
-    def _receive_frame(self):
-        try:
-            self.stream.read_bytes(2, self._on_frame_start)
-        except StreamClosedError:
-            self._abort()
-
-    def _on_frame_start(self, data):
-        self._wire_bytes_in += len(data)
-        header, payloadlen = struct.unpack("BB", data)
-        self._final_frame = header & self.FIN
-        reserved_bits = header & self.RSV_MASK
-        self._frame_opcode = header & self.OPCODE_MASK
-        self._frame_opcode_is_control = self._frame_opcode & 0x8
-        if self._decompressor is not None and self._frame_opcode != 0:
-            self._frame_compressed = bool(reserved_bits & self.RSV1)
-            reserved_bits &= ~self.RSV1
-        if reserved_bits:
-            # client is using as-yet-undefined extensions; abort
-            self._abort()
-            return
-        self._masked_frame = bool(payloadlen & 0x80)
-        payloadlen = payloadlen & 0x7f
-        if self._frame_opcode_is_control and payloadlen >= 126:
-            # control frames must have payload < 126
-            self._abort()
-            return
-        try:
-            if payloadlen < 126:
-                self._frame_length = payloadlen
-                if self._masked_frame:
-                    self.stream.read_bytes(4, self._on_masking_key)
-                else:
-                    self._read_frame_data(False)
-            elif payloadlen == 126:
-                self.stream.read_bytes(2, self._on_frame_length_16)
-            elif payloadlen == 127:
-                self.stream.read_bytes(8, self._on_frame_length_64)
-        except StreamClosedError:
-            self._abort()
-
-    def _read_frame_data(self, masked):
-        new_len = self._frame_length
-        if self._fragmented_message_buffer is not None:
-            new_len += len(self._fragmented_message_buffer)
-        if new_len > (self.handler.max_message_size or 10 * 1024 * 1024):
-            self.close(1009, "message too big")
-            return
-        self.stream.read_bytes(
-            self._frame_length,
-            self._on_masked_frame_data if masked else self._on_frame_data)
-
-    def _on_frame_length_16(self, data):
-        self._wire_bytes_in += len(data)
-        self._frame_length = struct.unpack("!H", data)[0]
-        try:
-            if self._masked_frame:
-                self.stream.read_bytes(4, self._on_masking_key)
-            else:
-                self._read_frame_data(False)
-        except StreamClosedError:
-            self._abort()
-
-    def _on_frame_length_64(self, data):
-        self._wire_bytes_in += len(data)
-        self._frame_length = struct.unpack("!Q", data)[0]
-        try:
-            if self._masked_frame:
-                self.stream.read_bytes(4, self._on_masking_key)
-            else:
-                self._read_frame_data(False)
-        except StreamClosedError:
-            self._abort()
-
-    def _on_masking_key(self, data):
-        self._wire_bytes_in += len(data)
-        self._frame_mask = data
-        try:
-            self._read_frame_data(True)
-        except StreamClosedError:
-            self._abort()
-
-    def _on_masked_frame_data(self, data):
-        # Don't touch _wire_bytes_in; we'll do it in _on_frame_data.
-        self._on_frame_data(_websocket_mask(self._frame_mask, data))
-
-    def _on_frame_data(self, data):
-        handled_future = None
-
-        self._wire_bytes_in += len(data)
-        if self._frame_opcode_is_control:
-            # control frames may be interleaved with a series of fragmented
-            # data frames, so control frames must not interact with
-            # self._fragmented_*
-            if not self._final_frame:
-                # control frames must not be fragmented
-                self._abort()
-                return
-            opcode = self._frame_opcode
-        elif self._frame_opcode == 0:  # continuation frame
-            if self._fragmented_message_buffer is None:
-                # nothing to continue
-                self._abort()
-                return
-            self._fragmented_message_buffer += data
-            if self._final_frame:
-                opcode = self._fragmented_message_opcode
-                data = self._fragmented_message_buffer
-                self._fragmented_message_buffer = None
-        else:  # start of new data message
-            if self._fragmented_message_buffer is not None:
-                # can't start new message until the old one is finished
-                self._abort()
-                return
-            if self._final_frame:
-                opcode = self._frame_opcode
-            else:
-                self._fragmented_message_opcode = self._frame_opcode
-                self._fragmented_message_buffer = data
-
-        if self._final_frame:
-            handled_future = self._handle_message(opcode, data)
-
-        if not self.client_terminated:
-            if handled_future:
-                # on_message is a coroutine, process more frames once it's done.
-                handled_future.add_done_callback(
-                    lambda future: self._receive_frame())
-            else:
-                self._receive_frame()
-
-    def _handle_message(self, opcode, data):
-        """Execute on_message, returning its Future if it is a coroutine."""
-        if self.client_terminated:
-            return
-
-        if self._frame_compressed:
-            data = self._decompressor.decompress(data)
-
-        if opcode == 0x1:
-            # UTF-8 data
-            self._message_bytes_in += len(data)
-            try:
-                decoded = data.decode("utf-8")
-            except UnicodeDecodeError:
-                self._abort()
-                return
-            return self._run_callback(self.handler.on_message, decoded)
-        elif opcode == 0x2:
-            # Binary data
-            self._message_bytes_in += len(data)
-            return self._run_callback(self.handler.on_message, data)
-        elif opcode == 0x8:
-            # Close
-            self.client_terminated = True
-            if len(data) >= 2:
-                self.handler.close_code = struct.unpack('>H', data[:2])[0]
-            if len(data) > 2:
-                self.handler.close_reason = to_unicode(data[2:])
-            # Echo the received close code, if any (RFC 6455 section 5.5.1).
-            self.close(self.handler.close_code)
-        elif opcode == 0x9:
-            # Ping
-            self._write_frame(True, 0xA, data)
-            self._run_callback(self.handler.on_ping, data)
-        elif opcode == 0xA:
-            # Pong
-            self.last_pong = IOLoop.current().time()
-            return self._run_callback(self.handler.on_pong, data)
-        else:
-            self._abort()
-
-    def close(self, code=None, reason=None):
-        """Closes the WebSocket connection."""
-        if not self.server_terminated:
-            if not self.stream.closed():
-                if code is None and reason is not None:
-                    code = 1000  # "normal closure" status code
-                if code is None:
-                    close_data = b''
-                else:
-                    close_data = struct.pack('>H', code)
-                if reason is not None:
-                    close_data += utf8(reason)
-                self._write_frame(True, 0x8, close_data)
-            self.server_terminated = True
-        if self.client_terminated:
-            if self._waiting is not None:
-                self.stream.io_loop.remove_timeout(self._waiting)
-                self._waiting = None
-            self.stream.close()
-        elif self._waiting is None:
-            # Give the client a few seconds to complete a clean shutdown,
-            # otherwise just close the connection.
-            self._waiting = self.stream.io_loop.add_timeout(
-                self.stream.io_loop.time() + 5, self._abort)
-
-    @property
-    def ping_interval(self):
-        interval = self.handler.ping_interval
-        if interval is not None:
-            return interval
-        return 0
-
-    @property
-    def ping_timeout(self):
-        timeout = self.handler.ping_timeout
-        if timeout is not None:
-            return timeout
-        return max(3 * self.ping_interval, 30)
-
-    def start_pinging(self):
-        """Start sending periodic pings to keep the connection alive"""
-        if self.ping_interval > 0:
-            self.last_ping = self.last_pong = IOLoop.current().time()
-            self.ping_callback = PeriodicCallback(
-                self.periodic_ping, self.ping_interval * 1000)
-            self.ping_callback.start()
-
-    def periodic_ping(self):
-        """Send a ping to keep the websocket alive
-
-        Called periodically if the websocket_ping_interval is set and non-zero.
-        """
-        if self.stream.closed() and self.ping_callback is not None:
-            self.ping_callback.stop()
-            return
-
-        # Check for timeout on pong. Make sure that we really have
-        # sent a recent ping in case the machine with both server and
-        # client has been suspended since the last ping.
-        now = IOLoop.current().time()
-        since_last_pong = now - self.last_pong
-        since_last_ping = now - self.last_ping
-        if (since_last_ping < 2 * self.ping_interval and
-                since_last_pong > self.ping_timeout):
-            self.close()
-            return
-
-        self.write_ping(b'')
-        self.last_ping = now
-
-
-class WebSocketClientConnection(simple_httpclient._HTTPConnection):
-    """WebSocket client connection.
-
-    This class should not be instantiated directly; use the
-    `websocket_connect` function instead.
-    """
-    def __init__(self, io_loop, request, on_message_callback=None,
-                 compression_options=None, ping_interval=None, ping_timeout=None,
-                 max_message_size=None):
-        self.compression_options = compression_options
-        self.connect_future = TracebackFuture()
-        self.protocol = None
-        self.read_future = None
-        self.read_queue = collections.deque()
-        self.key = base64.b64encode(os.urandom(16))
-        self._on_message_callback = on_message_callback
-        self.close_code = self.close_reason = None
-        self.ping_interval = ping_interval
-        self.ping_timeout = ping_timeout
-        self.max_message_size = max_message_size
-
-        scheme, sep, rest = request.url.partition(':')
-        scheme = {'ws': 'http', 'wss': 'https'}[scheme]
-        request.url = scheme + sep + rest
-        request.headers.update({
-            'Upgrade': 'websocket',
-            'Connection': 'Upgrade',
-            'Sec-WebSocket-Key': self.key,
-            'Sec-WebSocket-Version': '13',
-        })
-        if self.compression_options is not None:
-            # Always offer to let the server set our max_wbits (and even though
-            # we don't offer it, we will accept a client_no_context_takeover
-            # from the server).
-            # TODO: set server parameters for deflate extension
-            # if requested in self.compression_options.
-            request.headers['Sec-WebSocket-Extensions'] = (
-                'permessage-deflate; client_max_window_bits')
-
-        self.tcp_client = TCPClient(io_loop=io_loop)
-        super(WebSocketClientConnection, self).__init__(
-            io_loop, None, request, lambda: None, self._on_http_response,
-            104857600, self.tcp_client, 65536, 104857600)
-
-    def close(self, code=None, reason=None):
-        """Closes the websocket connection.
-
-        ``code`` and ``reason`` are documented under
-        `WebSocketHandler.close`.
-
-        .. versionadded:: 3.2
-
-        .. versionchanged:: 4.0
-
-           Added the ``code`` and ``reason`` arguments.
-        """
-        if self.protocol is not None:
-            self.protocol.close(code, reason)
-            self.protocol = None
-
-    def on_connection_close(self):
-        if not self.connect_future.done():
-            self.connect_future.set_exception(StreamClosedError())
-        self.on_message(None)
-        self.tcp_client.close()
-        super(WebSocketClientConnection, self).on_connection_close()
-
-    def _on_http_response(self, response):
-        if not self.connect_future.done():
-            if response.error:
-                self.connect_future.set_exception(response.error)
-            else:
-                self.connect_future.set_exception(WebSocketError(
-                    "Non-websocket response"))
-
-    def headers_received(self, start_line, headers):
-        if start_line.code != 101:
-            return super(WebSocketClientConnection, self).headers_received(
-                start_line, headers)
-
-        self.headers = headers
-        self.protocol = self.get_websocket_protocol()
-        self.protocol._process_server_headers(self.key, self.headers)
-        self.protocol.start_pinging()
-        self.protocol._receive_frame()
-
-        if self._timeout is not None:
-            self.io_loop.remove_timeout(self._timeout)
-            self._timeout = None
-
-        self.stream = self.connection.detach()
-        self.stream.set_close_callback(self.on_connection_close)
-        # Once we've taken over the connection, clear the final callback
-        # we set on the http request.  This deactivates the error handling
-        # in simple_httpclient that would otherwise interfere with our
-        # ability to see exceptions.
-        self.final_callback = None
-
-        self.connect_future.set_result(self)
-
-    def write_message(self, message, binary=False):
-        """Sends a message to the WebSocket server."""
-        return self.protocol.write_message(message, binary)
-
-    def read_message(self, callback=None):
-        """Reads a message from the WebSocket server.
-
-        If on_message_callback was specified at WebSocket
-        initialization, this function will never return messages
-
-        Returns a future whose result is the message, or None
-        if the connection is closed.  If a callback argument
-        is given it will be called with the future when it is
-        ready.
-        """
-        assert self.read_future is None
-        future = TracebackFuture()
-        if self.read_queue:
-            future.set_result(self.read_queue.popleft())
-        else:
-            self.read_future = future
-        if callback is not None:
-            self.io_loop.add_future(future, callback)
-        return future
-
-    def on_message(self, message):
-        if self._on_message_callback:
-            self._on_message_callback(message)
-        elif self.read_future is not None:
-            self.read_future.set_result(message)
-            self.read_future = None
-        else:
-            self.read_queue.append(message)
-
-    def on_pong(self, data):
-        pass
-
-    def on_ping(self, data):
-        pass
-
-    def get_websocket_protocol(self):
-        return WebSocketProtocol13(self, mask_outgoing=True,
-                                   compression_options=self.compression_options)
-
-
-def websocket_connect(url, io_loop=None, callback=None, connect_timeout=None,
-                      on_message_callback=None, compression_options=None,
-                      ping_interval=None, ping_timeout=None,
-                      max_message_size=None):
-    """Client-side websocket support.
-
-    Takes a url and returns a Future whose result is a
-    `WebSocketClientConnection`.
-
-    ``compression_options`` is interpreted in the same way as the
-    return value of `.WebSocketHandler.get_compression_options`.
-
-    The connection supports two styles of operation. In the coroutine
-    style, the application typically calls
-    `~.WebSocketClientConnection.read_message` in a loop::
-
-        conn = yield websocket_connect(url)
-        while True:
-            msg = yield conn.read_message()
-            if msg is None: break
-            # Do something with msg
-
-    In the callback style, pass an ``on_message_callback`` to
-    ``websocket_connect``. In both styles, a message of ``None``
-    indicates that the connection has been closed.
-
-    .. versionchanged:: 3.2
-       Also accepts ``HTTPRequest`` objects in place of urls.
-
-    .. versionchanged:: 4.1
-       Added ``compression_options`` and ``on_message_callback``.
-       The ``io_loop`` argument is deprecated.
-
-    .. versionchanged:: 4.5
-       Added the ``ping_interval``, ``ping_timeout``, and ``max_message_size``
-       arguments, which have the same meaning as in `WebSocketHandler`.
-    """
-    if io_loop is None:
-        io_loop = IOLoop.current()
-    if isinstance(url, httpclient.HTTPRequest):
-        assert connect_timeout is None
-        request = url
-        # Copy and convert the headers dict/object (see comments in
-        # AsyncHTTPClient.fetch)
-        request.headers = httputil.HTTPHeaders(request.headers)
-    else:
-        request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout)
-    request = httpclient._RequestProxy(
-        request, httpclient.HTTPRequest._DEFAULTS)
-    conn = WebSocketClientConnection(io_loop, request,
-                                     on_message_callback=on_message_callback,
-                                     compression_options=compression_options,
-                                     ping_interval=ping_interval,
-                                     ping_timeout=ping_timeout,
-                                     max_message_size=max_message_size)
-    if callback is not None:
-        io_loop.add_future(conn.connect_future, callback)
-    return conn.connect_future
diff --git a/salt/ext/tornado/wsgi.py b/salt/ext/tornado/wsgi.py
deleted file mode 100644
index 31ed0f4323c..00000000000
--- a/salt/ext/tornado/wsgi.py
+++ /dev/null
@@ -1,359 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2009 Facebook
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""WSGI support for the Tornado web framework.
-
-WSGI is the Python standard for web servers, and allows for interoperability
-between Tornado and other Python web frameworks and servers.  This module
-provides WSGI support in two ways:
-
-* `WSGIAdapter` converts a `tornado.web.Application` to the WSGI application
-  interface.  This is useful for running a Tornado app on another
-  HTTP server, such as Google App Engine.  See the `WSGIAdapter` class
-  documentation for limitations that apply.
-* `WSGIContainer` lets you run other WSGI applications and frameworks on the
-  Tornado HTTP server.  For example, with this class you can mix Django
-  and Tornado handlers in a single server.
-"""
-# pylint: skip-file
-
-from __future__ import absolute_import, division, print_function
-
-import sys
-from io import BytesIO
-import salt.ext.tornado as tornado
-
-from salt.ext.tornado.concurrent import Future
-from salt.ext.tornado import escape
-from salt.ext.tornado import httputil
-from salt.ext.tornado.log import access_log
-from salt.ext.tornado import web
-from salt.ext.tornado.escape import native_str
-from salt.ext.tornado.util import unicode_type, PY3
-
-
-if PY3:
-    import urllib.parse as urllib_parse  # py3
-else:
-    import urllib as urllib_parse
-
-# PEP 3333 specifies that WSGI on python 3 generally deals with byte strings
-# that are smuggled inside objects of type unicode (via the latin1 encoding).
-# These functions are like those in the tornado.escape module, but defined
-# here to minimize the temptation to use them in non-wsgi contexts.
-if str is unicode_type:
-    def to_wsgi_str(s):
-        assert isinstance(s, bytes)
-        return s.decode('latin1')
-
-    def from_wsgi_str(s):
-        assert isinstance(s, str)
-        return s.encode('latin1')
-else:
-    def to_wsgi_str(s):
-        assert isinstance(s, bytes)
-        return s
-
-    def from_wsgi_str(s):
-        assert isinstance(s, str)
-        return s
-
-
-class WSGIApplication(web.Application):
-    """A WSGI equivalent of `tornado.web.Application`.
-
-    .. deprecated:: 4.0
-
-       Use a regular `.Application` and wrap it in `WSGIAdapter` instead.
-    """
-    def __call__(self, environ, start_response):
-        return WSGIAdapter(self)(environ, start_response)
-
-
-# WSGI has no facilities for flow control, so just return an already-done
-# Future when the interface requires it.
-_dummy_future = Future()
-_dummy_future.set_result(None)
-
-
-class _WSGIConnection(httputil.HTTPConnection):
-    def __init__(self, method, start_response, context):
-        self.method = method
-        self.start_response = start_response
-        self.context = context
-        self._write_buffer = []
-        self._finished = False
-        self._expected_content_remaining = None
-        self._error = None
-
-    def set_close_callback(self, callback):
-        # WSGI has no facility for detecting a closed connection mid-request,
-        # so we can simply ignore the callback.
-        pass
-
-    def write_headers(self, start_line, headers, chunk=None, callback=None):
-        if self.method == 'HEAD':
-            self._expected_content_remaining = 0
-        elif 'Content-Length' in headers:
-            self._expected_content_remaining = int(headers['Content-Length'])
-        else:
-            self._expected_content_remaining = None
-        self.start_response(
-            '%s %s' % (start_line.code, start_line.reason),
-            [(native_str(k), native_str(v)) for (k, v) in headers.get_all()])
-        if chunk is not None:
-            self.write(chunk, callback)
-        elif callback is not None:
-            callback()
-        return _dummy_future
-
-    def write(self, chunk, callback=None):
-        if self._expected_content_remaining is not None:
-            self._expected_content_remaining -= len(chunk)
-            if self._expected_content_remaining < 0:
-                self._error = httputil.HTTPOutputError(
-                    "Tried to write more data than Content-Length")
-                raise self._error
-        self._write_buffer.append(chunk)
-        if callback is not None:
-            callback()
-        return _dummy_future
-
-    def finish(self):
-        if (self._expected_content_remaining is not None and
-                self._expected_content_remaining != 0):
-            self._error = httputil.HTTPOutputError(
-                "Tried to write %d bytes less than Content-Length" %
-                self._expected_content_remaining)
-            raise self._error
-        self._finished = True
-
-
-class _WSGIRequestContext(object):
-    def __init__(self, remote_ip, protocol):
-        self.remote_ip = remote_ip
-        self.protocol = protocol
-
-    def __str__(self):
-        return self.remote_ip
-
-
-class WSGIAdapter(object):
-    """Converts a `tornado.web.Application` instance into a WSGI application.
-
-    Example usage::
-
-        import tornado.web
-        import tornado.wsgi
-        import wsgiref.simple_server
-
-        class MainHandler(tornado.web.RequestHandler):
-            def get(self):
-                self.write("Hello, world")
-
-        if __name__ == "__main__":
-            application = tornado.web.Application([
-                (r"/", MainHandler),
-            ])
-            wsgi_app = tornado.wsgi.WSGIAdapter(application)
-            server = wsgiref.simple_server.make_server('', 8888, wsgi_app)
-            server.serve_forever()
-
-    See the `appengine demo
-    `_
-    for an example of using this module to run a Tornado app on Google
-    App Engine.
-
-    In WSGI mode asynchronous methods are not supported.  This means
-    that it is not possible to use `.AsyncHTTPClient`, or the
-    `tornado.auth` or `tornado.websocket` modules.
-
-    .. versionadded:: 4.0
-    """
-    def __init__(self, application):
-        if isinstance(application, WSGIApplication):
-            self.application = lambda request: web.Application.__call__(
-                application, request)
-        else:
-            self.application = application
-
-    def __call__(self, environ, start_response):
-        method = environ["REQUEST_METHOD"]
-        uri = urllib_parse.quote(from_wsgi_str(environ.get("SCRIPT_NAME", "")))
-        uri += urllib_parse.quote(from_wsgi_str(environ.get("PATH_INFO", "")))
-        if environ.get("QUERY_STRING"):
-            uri += "?" + environ["QUERY_STRING"]
-        headers = httputil.HTTPHeaders()
-        if environ.get("CONTENT_TYPE"):
-            headers["Content-Type"] = environ["CONTENT_TYPE"]
-        if environ.get("CONTENT_LENGTH"):
-            headers["Content-Length"] = environ["CONTENT_LENGTH"]
-        for key in environ:
-            if key.startswith("HTTP_"):
-                headers[key[5:].replace("_", "-")] = environ[key]
-        if headers.get("Content-Length"):
-            body = environ["wsgi.input"].read(
-                int(headers["Content-Length"]))
-        else:
-            body = b""
-        protocol = environ["wsgi.url_scheme"]
-        remote_ip = environ.get("REMOTE_ADDR", "")
-        if environ.get("HTTP_HOST"):
-            host = environ["HTTP_HOST"]
-        else:
-            host = environ["SERVER_NAME"]
-        connection = _WSGIConnection(method, start_response,
-                                     _WSGIRequestContext(remote_ip, protocol))
-        request = httputil.HTTPServerRequest(
-            method, uri, "HTTP/1.1", headers=headers, body=body,
-            host=host, connection=connection)
-        request._parse_body()
-        self.application(request)
-        if connection._error:
-            raise connection._error
-        if not connection._finished:
-            raise Exception("request did not finish synchronously")
-        return connection._write_buffer
-
-
-class WSGIContainer(object):
-    r"""Makes a WSGI-compatible function runnable on Tornado's HTTP server.
-
-    .. warning::
-
-       WSGI is a *synchronous* interface, while Tornado's concurrency model
-       is based on single-threaded asynchronous execution.  This means that
-       running a WSGI app with Tornado's `WSGIContainer` is *less scalable*
-       than running the same app in a multi-threaded WSGI server like
-       ``gunicorn`` or ``uwsgi``.  Use `WSGIContainer` only when there are
-       benefits to combining Tornado and WSGI in the same process that
-       outweigh the reduced scalability.
-
-    Wrap a WSGI function in a `WSGIContainer` and pass it to `.HTTPServer` to
-    run it. For example::
-
-        def simple_app(environ, start_response):
-            status = "200 OK"
-            response_headers = [("Content-type", "text/plain")]
-            start_response(status, response_headers)
-            return ["Hello world!\n"]
-
-        container = tornado.wsgi.WSGIContainer(simple_app)
-        http_server = tornado.httpserver.HTTPServer(container)
-        http_server.listen(8888)
-        tornado.ioloop.IOLoop.current().start()
-
-    This class is intended to let other frameworks (Django, web.py, etc)
-    run on the Tornado HTTP server and I/O loop.
-
-    The `tornado.web.FallbackHandler` class is often useful for mixing
-    Tornado and WSGI apps in the same server.  See
-    https://github.com/bdarnell/django-tornado-demo for a complete example.
-    """
-    def __init__(self, wsgi_application):
-        self.wsgi_application = wsgi_application
-
-    def __call__(self, request):
-        data = {}
-        response = []
-
-        def start_response(status, response_headers, exc_info=None):
-            data["status"] = status
-            data["headers"] = response_headers
-            return response.append
-        app_response = self.wsgi_application(
-            WSGIContainer.environ(request), start_response)
-        try:
-            response.extend(app_response)
-            body = b"".join(response)
-        finally:
-            if hasattr(app_response, "close"):
-                app_response.close()
-        if not data:
-            raise Exception("WSGI app did not call start_response")
-
-        status_code, reason = data["status"].split(' ', 1)
-        status_code = int(status_code)
-        headers = data["headers"]
-        header_set = set(k.lower() for (k, v) in headers)
-        body = escape.utf8(body)
-        if status_code != 304:
-            if "content-length" not in header_set:
-                headers.append(("Content-Length", str(len(body))))
-            if "content-type" not in header_set:
-                headers.append(("Content-Type", "text/html; charset=UTF-8"))
-        if "server" not in header_set:
-            headers.append(("Server", "TornadoServer/%s" % salt.ext.tornado.version))
-
-        start_line = httputil.ResponseStartLine("HTTP/1.1", status_code, reason)
-        header_obj = httputil.HTTPHeaders()
-        for key, value in headers:
-            header_obj.add(key, value)
-        request.connection.write_headers(start_line, header_obj, chunk=body)
-        request.connection.finish()
-        self._log(status_code, request)
-
-    @staticmethod
-    def environ(request):
-        """Converts a `tornado.httputil.HTTPServerRequest` to a WSGI environment.
-        """
-        hostport = request.host.split(":")
-        if len(hostport) == 2:
-            host = hostport[0]
-            port = int(hostport[1])
-        else:
-            host = request.host
-            port = 443 if request.protocol == "https" else 80
-        environ = {
-            "REQUEST_METHOD": request.method,
-            "SCRIPT_NAME": "",
-            "PATH_INFO": to_wsgi_str(escape.url_unescape(
-                request.path, encoding=None, plus=False)),
-            "QUERY_STRING": request.query,
-            "REMOTE_ADDR": request.remote_ip,
-            "SERVER_NAME": host,
-            "SERVER_PORT": str(port),
-            "SERVER_PROTOCOL": request.version,
-            "wsgi.version": (1, 0),
-            "wsgi.url_scheme": request.protocol,
-            "wsgi.input": BytesIO(escape.utf8(request.body)),
-            "wsgi.errors": sys.stderr,
-            "wsgi.multithread": False,
-            "wsgi.multiprocess": True,
-            "wsgi.run_once": False,
-        }
-        if "Content-Type" in request.headers:
-            environ["CONTENT_TYPE"] = request.headers.pop("Content-Type")
-        if "Content-Length" in request.headers:
-            environ["CONTENT_LENGTH"] = request.headers.pop("Content-Length")
-        for key, value in request.headers.items():
-            environ["HTTP_" + key.replace("-", "_").upper()] = value
-        return environ
-
-    def _log(self, status_code, request):
-        if status_code < 400:
-            log_method = access_log.info
-        elif status_code < 500:
-            log_method = access_log.warning
-        else:
-            log_method = access_log.error
-        request_time = 1000.0 * request.request_time()
-        summary = request.method + " " + request.uri + " (" + \
-            request.remote_ip + ")"
-        log_method("%d %s %.2fms", status_code, summary, request_time)
-
-
-HTTPRequest = httputil.HTTPServerRequest
diff --git a/salt/fileclient.py b/salt/fileclient.py
index f01a86dd0d4..04e004a8b53 100644
--- a/salt/fileclient.py
+++ b/salt/fileclient.py
@@ -12,6 +12,8 @@ import string
 import urllib.error
 import urllib.parse
 
+from tornado.httputil import HTTPHeaders, HTTPInputError, parse_response_start_line
+
 import salt.channel.client
 import salt.client
 import salt.crypt
@@ -32,11 +34,6 @@ import salt.utils.url
 import salt.utils.verify
 import salt.utils.versions
 from salt.exceptions import CommandExecutionError, MinionError
-from salt.ext.tornado.httputil import (
-    HTTPHeaders,
-    HTTPInputError,
-    parse_response_start_line,
-)
 from salt.utils.openstack.swift import SaltSwift
 
 log = logging.getLogger(__name__)
diff --git a/salt/master.py b/salt/master.py
index 5642ccda35d..33c8de076aa 100644
--- a/salt/master.py
+++ b/salt/master.py
@@ -15,6 +15,8 @@ import sys
 import threading
 import time
 
+import tornado.gen
+
 import salt.acl
 import salt.auth
 import salt.channel.server
@@ -25,7 +27,6 @@ import salt.daemons.masterapi
 import salt.defaults.exitcodes
 import salt.engines
 import salt.exceptions
-import salt.ext.tornado.gen
 import salt.key
 import salt.minion
 import salt.payload
@@ -998,7 +999,7 @@ class MWorker(salt.utils.process.SignalHandlingProcess):
         """
         Bind to the local port
         """
-        self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+        self.io_loop = tornado.ioloop.IOLoop()
         self.io_loop.make_current()
         for req_channel in self.req_channels:
             req_channel.post_fork(
@@ -1010,7 +1011,7 @@ class MWorker(salt.utils.process.SignalHandlingProcess):
             # Tornado knows what to do
             pass
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _handle_payload(self, payload):
         """
         The _handle_payload method is the key method used to figure out what
@@ -1035,7 +1036,7 @@ class MWorker(salt.utils.process.SignalHandlingProcess):
         key = payload["enc"]
         load = payload["load"]
         ret = {"aes": self._handle_aes, "clear": self._handle_clear}[key](load)
-        raise salt.ext.tornado.gen.Return(ret)
+        raise tornado.gen.Return(ret)
 
     def _post_stats(self, start, cmd):
         """
diff --git a/salt/metaproxy/deltaproxy.py b/salt/metaproxy/deltaproxy.py
index de07a041956..a9a7fa50a18 100644
--- a/salt/metaproxy/deltaproxy.py
+++ b/salt/metaproxy/deltaproxy.py
@@ -11,6 +11,9 @@ import threading
 import traceback
 import types
 
+import tornado.gen  # pylint: disable=F0401
+import tornado.ioloop  # pylint: disable=F0401
+
 import salt
 import salt._logging
 import salt.beacons
@@ -20,8 +23,6 @@ import salt.config
 import salt.crypt
 import salt.defaults.exitcodes
 import salt.engines
-import salt.ext.tornado.gen  # pylint: disable=F0401
-import salt.ext.tornado.ioloop  # pylint: disable=F0401
 import salt.loader
 import salt.minion
 import salt.payload
@@ -59,7 +60,7 @@ from salt.utils.process import SignalHandlingProcess, default_signals
 log = logging.getLogger(__name__)
 
 
-@salt.ext.tornado.gen.coroutine
+@tornado.gen.coroutine
 def post_master_init(self, master):
     """
     Function to finish init after a deltaproxy proxy
@@ -352,7 +353,7 @@ def post_master_init(self, master):
             )
 
         try:
-            results = yield salt.ext.tornado.gen.multi(waitfor)
+            results = yield tornado.gen.multi(waitfor)
         except Exception as exc:  # pylint: disable=broad-except
             log.error("Errors loading sub proxies")
 
@@ -405,7 +406,7 @@ def post_master_init(self, master):
     self.ready = True
 
 
-@salt.ext.tornado.gen.coroutine
+@tornado.gen.coroutine
 def subproxy_post_master_init(minion_id, uid, opts, main_proxy, main_utils):
     """
     Function to finish init after a deltaproxy proxy
@@ -577,9 +578,7 @@ def subproxy_post_master_init(minion_id, uid, opts, main_proxy, main_utils):
             "__proxy_keepalive", persist=True, fire_event=False
         )
 
-    raise salt.ext.tornado.gen.Return(
-        {"proxy_minion": _proxy_minion, "proxy_opts": proxyopts}
-    )
+    raise tornado.gen.Return({"proxy_minion": _proxy_minion, "proxy_opts": proxyopts})
 
 
 def target(cls, minion_instance, opts, data, connected):
@@ -1052,7 +1051,7 @@ def handle_decoded_payload(self, data):
                     data["jid"],
                 )
                 once_logged = True
-            yield salt.ext.tornado.gen.sleep(0.5)
+            yield tornado.gen.sleep(0.5)
             process_count = self.subprocess_list.count
 
     # We stash an instance references to allow for the socket
diff --git a/salt/metaproxy/proxy.py b/salt/metaproxy/proxy.py
index 039081cf5f1..ace2260cf62 100644
--- a/salt/metaproxy/proxy.py
+++ b/salt/metaproxy/proxy.py
@@ -9,6 +9,9 @@ import threading
 import traceback
 import types
 
+import tornado.gen  # pylint: disable=F0401
+import tornado.ioloop  # pylint: disable=F0401
+
 import salt
 import salt.beacons
 import salt.cli.daemons
@@ -16,8 +19,6 @@ import salt.client
 import salt.crypt
 import salt.defaults.exitcodes
 import salt.engines
-import salt.ext.tornado.gen  # pylint: disable=F0401
-import salt.ext.tornado.ioloop  # pylint: disable=F0401
 import salt.loader
 import salt.minion
 import salt.payload
@@ -55,7 +56,7 @@ from salt.utils.process import SignalHandlingProcess, default_signals
 log = logging.getLogger(__name__)
 
 
-@salt.ext.tornado.gen.coroutine
+@tornado.gen.coroutine
 def post_master_init(self, master):
     """
     Function to finish init after a proxy
@@ -804,7 +805,7 @@ def handle_decoded_payload(self, data):
                 "Maximum number of processes reached while executing jid %s, waiting...",
                 data["jid"],
             )
-            yield salt.ext.tornado.gen.sleep(10)
+            yield tornado.gen.sleep(10)
             process_count = len(salt.utils.minion.running(self.opts))
 
     # We stash an instance references to allow for the socket
diff --git a/salt/minion.py b/salt/minion.py
index 64291459e4a..104eccb10f5 100644
--- a/salt/minion.py
+++ b/salt/minion.py
@@ -17,6 +17,10 @@ import time
 import traceback
 import types
 
+import tornado
+import tornado.gen
+import tornado.ioloop
+
 import salt
 import salt.beacons
 import salt.channel.client
@@ -26,9 +30,6 @@ import salt.crypt
 import salt.defaults.events
 import salt.defaults.exitcodes
 import salt.engines
-import salt.ext.tornado
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
 import salt.loader
 import salt.loader.lazy
 import salt.payload
@@ -512,7 +513,7 @@ class MinionBase:
                 )  # pylint: disable=no-member
         return []
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def eval_master(self, opts, timeout=60, safe=True, failed=False, failback=False):
         """
         Evaluates and returns a tuple of the current master address and the pub_channel.
@@ -533,7 +534,7 @@ class MinionBase:
         if opts["master_type"] == "disable":
             log.warning("Master is set to disable, skipping connection")
             self.connected = False
-            raise salt.ext.tornado.gen.Return((None, None))
+            raise tornado.gen.Return((None, None))
 
         # Run masters discovery over SSDP. This may modify the whole configuration,
         # depending of the networking and sets of masters.
@@ -703,7 +704,7 @@ class MinionBase:
                 if attempts != 0:
                     # Give up a little time between connection attempts
                     # to allow the IOLoop to run any other scheduled tasks.
-                    yield salt.ext.tornado.gen.sleep(opts["acceptance_wait_time"])
+                    yield tornado.gen.sleep(opts["acceptance_wait_time"])
                 attempts += 1
                 if tries > 0:
                     log.debug("Connecting to master. Attempt %s of %s", attempts, tries)
@@ -772,7 +773,7 @@ class MinionBase:
                 else:
                     self.tok = pub_channel.auth.gen_token(b"salt")
                     self.connected = True
-                    raise salt.ext.tornado.gen.Return((opts["master"], pub_channel))
+                    raise tornado.gen.Return((opts["master"], pub_channel))
 
         # single master sign in
         else:
@@ -786,7 +787,7 @@ class MinionBase:
                 if attempts != 0:
                     # Give up a little time between connection attempts
                     # to allow the IOLoop to run any other scheduled tasks.
-                    yield salt.ext.tornado.gen.sleep(opts["acceptance_wait_time"])
+                    yield tornado.gen.sleep(opts["acceptance_wait_time"])
                 attempts += 1
                 if tries > 0:
                     log.debug("Connecting to master. Attempt %s of %s", attempts, tries)
@@ -818,7 +819,7 @@ class MinionBase:
                         yield pub_channel.connect()
                     self.tok = pub_channel.auth.gen_token(b"salt")
                     self.connected = True
-                    raise salt.ext.tornado.gen.Return((opts["master"], pub_channel))
+                    raise tornado.gen.Return((opts["master"], pub_channel))
                 except SaltClientError:
                     if pub_channel:
                         pub_channel.close()
@@ -924,7 +925,7 @@ class SMinion(MinionBase):
         if self.opts.get("file_client", "remote") == "remote" or self.opts.get(
             "use_master_when_local", False
         ):
-            io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+            io_loop = tornado.ioloop.IOLoop.current()
             io_loop.run_sync(lambda: self.eval_master(self.opts, failed=True))
         self.gen_modules(initial_load=True, context=context)
 
@@ -1027,7 +1028,7 @@ class MinionManager(MinionBase):
         self.minions = []
         self.jid_queue = []
 
-        self.io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+        self.io_loop = tornado.ioloop.IOLoop.current()
         self.process_manager = ProcessManager(name="MultiMinionProcessManager")
         self.io_loop.spawn_callback(
             self.process_manager.run, **{"asynchronous": True}
@@ -1053,7 +1054,7 @@ class MinionManager(MinionBase):
         self.event.subscribe("")
         self.event.set_event_handler(self.handle_event)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def handle_event(self, package):
         for minion in self.minions:
             minion.handle_event(package)
@@ -1117,7 +1118,7 @@ class MinionManager(MinionBase):
             self.io_loop.spawn_callback(self._connect_minion, minion)
         self.io_loop.call_later(timeout, self._check_minions)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _connect_minion(self, minion):
         """
         Create a minion, and asynchronously connect it to a master
@@ -1146,7 +1147,7 @@ class MinionManager(MinionBase):
                 last = time.time()
                 if auth_wait < self.max_auth_wait:
                     auth_wait += self.auth_wait
-                yield salt.ext.tornado.gen.sleep(auth_wait)  # TODO: log?
+                yield tornado.gen.sleep(auth_wait)  # TODO: log?
             except SaltMasterUnresolvableError:
                 err = (
                     "Master address: '{}' could not be resolved. Invalid or"
@@ -1250,7 +1251,7 @@ class Minion(MinionBase):
         self.periodic_callbacks = {}
 
         if io_loop is None:
-            self.io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+            self.io_loop = tornado.ioloop.IOLoop.current()
         else:
             self.io_loop = io_loop
 
@@ -1353,7 +1354,7 @@ class Minion(MinionBase):
         if timeout and self._sync_connect_master_success is False:
             raise SaltDaemonNotRunning("Failed to connect to the salt-master")
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def connect_master(self, failed=False):
         """
         Return a future which will complete when you are connected to a master
@@ -1376,14 +1377,14 @@ class Minion(MinionBase):
 
         yield self._post_master_init(master)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def handle_payload(self, payload, reply_func):
         self.payloads.append(payload)
         yield reply_func(payload)
         self.payload_ack.notify()
 
     # TODO: better name...
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _post_master_init(self, master):
         """
         Function to finish init after connecting to a master
@@ -1611,7 +1612,7 @@ class Minion(MinionBase):
                 load, "__master_req_channel_payload", timeout=timeout
             )
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _send_req_async(self, load, timeout):
         if self.opts["minion_sign_messages"]:
             log.trace("Signing event to be published onto the bus.")
@@ -1627,7 +1628,7 @@ class Minion(MinionBase):
             ret = yield event.fire_event_async(
                 load, "__master_req_channel_payload", timeout=timeout
             )
-            raise salt.ext.tornado.gen.Return(ret)
+            raise tornado.gen.Return(ret)
 
     def _fire_master(
         self,
@@ -1697,7 +1698,7 @@ class Minion(MinionBase):
             # pylint: enable=unexpected-keyword-arg
         return True
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _handle_decoded_payload(self, data):
         """
         Override this method if you wish to handle the decoded data
@@ -1755,7 +1756,7 @@ class Minion(MinionBase):
                     " waiting...",
                     data["jid"],
                 )
-                yield salt.ext.tornado.gen.sleep(10)
+                yield tornado.gen.sleep(10)
                 process_count = len(salt.utils.minion.running(self.opts))
 
         # We stash an instance references to allow for the socket
@@ -2487,7 +2488,7 @@ class Minion(MinionBase):
         return pillar_schedule
 
     # TODO: only allow one future in flight at a time?
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def pillar_refresh(self, force_refresh=False, clean_cache=False):
         """
         Refresh the pillar
@@ -2675,19 +2676,19 @@ class Minion(MinionBase):
                 log.warning("Unable to send mine data to master.")
                 return None
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def handle_event(self, package):
         """
         Handle an event from the epull_sock (all local minion events)
         """
         if not self.ready:
-            raise salt.ext.tornado.gen.Return()
+            raise tornado.gen.Return()
         tag, data = salt.utils.event.SaltEvent.unpack(package)
 
         if "proxy_target" in data and self.opts.get("metaproxy") == "deltaproxy":
             proxy_target = data["proxy_target"]
             if proxy_target not in self.deltaproxy_objs:
-                raise salt.ext.tornado.gen.Return()
+                raise tornado.gen.Return()
             _minion = self.deltaproxy_objs[proxy_target]
         else:
             _minion = self
@@ -2747,7 +2748,7 @@ class Minion(MinionBase):
                 and data["master"] != self.opts["master"]
             ):
                 # not mine master, ignore
-                raise salt.ext.tornado.gen.Return()
+                raise tornado.gen.Return()
             if tag.startswith(master_event(type="failback")):
                 # if the master failback event is not for the top master, raise an exception
                 if data["master"] != self.opts["master_list"][0]:
@@ -3058,7 +3059,7 @@ class Minion(MinionBase):
         """
         if name in self.periodic_callbacks:
             return False
-        self.periodic_callbacks[name] = salt.ext.tornado.ioloop.PeriodicCallback(
+        self.periodic_callbacks[name] = tornado.ioloop.PeriodicCallback(
             method,
             interval * 1000,
         )
@@ -3321,7 +3322,7 @@ class Syndic(Minion):
             load, timeout=timeout, tries=self.opts["return_retry_tries"]
         )
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _send_req_async(self, load, timeout):
         if self.opts["minion_sign_messages"]:
             log.trace("Signing event to be published onto the bus.")
@@ -3375,7 +3376,7 @@ class Syndic(Minion):
         # In the future, we could add support for some clearfuncs, but
         # the syndic currently has no need.
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def reconnect(self):
         if hasattr(self, "pub_channel"):
             self.pub_channel.on_recv(None)
@@ -3392,7 +3393,7 @@ class Syndic(Minion):
             self.pub_channel.on_recv(self._process_cmd_socket)
             log.info("Minion is ready to receive requests!")
 
-        raise salt.ext.tornado.gen.Return(self)
+        raise tornado.gen.Return(self)
 
     def destroy(self):
         """
@@ -3451,7 +3452,7 @@ class SyndicManager(MinionBase):
         self.jid_forward_cache = set()
 
         if io_loop is None:
-            self.io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+            self.io_loop = tornado.ioloop.IOLoop.current()
         else:
             self.io_loop = io_loop
 
@@ -3478,7 +3479,7 @@ class SyndicManager(MinionBase):
             s_opts["master"] = master
             self._syndics[master] = self._connect_syndic(s_opts)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _connect_syndic(self, opts):
         """
         Create a syndic, and asynchronously connect it to a master
@@ -3514,7 +3515,7 @@ class SyndicManager(MinionBase):
                 last = time.time()
                 if auth_wait < self.max_auth_wait:
                     auth_wait += self.auth_wait
-                yield salt.ext.tornado.gen.sleep(auth_wait)  # TODO: log?
+                yield tornado.gen.sleep(auth_wait)  # TODO: log?
             except (KeyboardInterrupt, SystemExit):  # pylint: disable=try-except-raise
                 raise
             except Exception:  # pylint: disable=broad-except
@@ -3525,7 +3526,7 @@ class SyndicManager(MinionBase):
                     exc_info=True,
                 )
 
-        raise salt.ext.tornado.gen.Return(syndic)
+        raise tornado.gen.Return(syndic)
 
     def _mark_master_dead(self, master):
         """
@@ -3657,7 +3658,7 @@ class SyndicManager(MinionBase):
         self.io_loop.add_future(future, self.reconnect_event_bus)
 
         # forward events every syndic_event_forward_timeout
-        self.forward_events = salt.ext.tornado.ioloop.PeriodicCallback(
+        self.forward_events = tornado.ioloop.PeriodicCallback(
             self._forward_events,
             self.opts["syndic_event_forward_timeout"] * 1000,
         )
@@ -3812,7 +3813,7 @@ class ProxyMinion(Minion):
     """
 
     # TODO: better name...
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _post_master_init(self, master):
         """
         Function to finish init after connecting to a master
@@ -3830,7 +3831,7 @@ class ProxyMinion(Minion):
         mp_call = _metaproxy_call(self.opts, "post_master_init")
         yield mp_call(self, master)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def subproxy_post_master_init(self, minion_id, uid):
         """
         Function to finish init for the sub proxies
@@ -3859,7 +3860,7 @@ class ProxyMinion(Minion):
         mp_call = _metaproxy_call(self.opts, "handle_payload")
         return mp_call(self, payload)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _handle_decoded_payload(self, data):
         mp_call = _metaproxy_call(self.opts, "handle_decoded_payload")
         return mp_call(self, data)
diff --git a/salt/netapi/rest_tornado/__init__.py b/salt/netapi/rest_tornado/__init__.py
index ff0db64f6ea..08c8c2027a2 100644
--- a/salt/netapi/rest_tornado/__init__.py
+++ b/salt/netapi/rest_tornado/__init__.py
@@ -13,9 +13,9 @@ log = logging.getLogger(__virtualname__)
 min_tornado_version = "4.0"
 has_tornado = False
 try:
-    import salt.ext.tornado
+    import tornado
 
-    if Version(salt.ext.tornado.version) >= Version(min_tornado_version):
+    if Version(tornado.version) >= Version(min_tornado_version):
         has_tornado = True
     else:
         log.error("rest_tornado requires at least tornado %s", min_tornado_version)
@@ -74,7 +74,7 @@ def get_application(opts):
             (formatted_events_pattern, saltnado_websockets.FormattedEventsHandler),
         ]
 
-    application = salt.ext.tornado.web.Application(paths, mod_opts.get("debug", False))
+    application = tornado.web.Application(paths, mod_opts.get("debug", False))
 
     application.opts = opts
     application.mod_opts = mod_opts
@@ -115,11 +115,9 @@ def start():
             ssl_opts.update({"keyfile": mod_opts["ssl_key"]})
         kwargs["ssl_options"] = ssl_opts
 
-    import salt.ext.tornado.httpserver
+    import tornado.httpserver
 
-    http_server = salt.ext.tornado.httpserver.HTTPServer(
-        get_application(__opts__), **kwargs
-    )
+    http_server = tornado.httpserver.HTTPServer(get_application(__opts__), **kwargs)
     try:
         http_server.bind(
             mod_opts["port"],
@@ -134,6 +132,6 @@ def start():
         raise SystemExit(1)
 
     try:
-        salt.ext.tornado.ioloop.IOLoop.current().start()
+        tornado.ioloop.IOLoop.current().start()
     except KeyboardInterrupt:
         raise SystemExit(0)
diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py
index e5838afce90..a57986d48b1 100644
--- a/salt/netapi/rest_tornado/saltnado.py
+++ b/salt/netapi/rest_tornado/saltnado.py
@@ -192,13 +192,15 @@ import time
 from collections import defaultdict
 from copy import copy
 
+import tornado.escape
+import tornado.gen
+import tornado.httpserver
+import tornado.ioloop
+import tornado.web
+from tornado.concurrent import Future
+
 import salt.auth
 import salt.client
-import salt.ext.tornado.escape
-import salt.ext.tornado.gen
-import salt.ext.tornado.httpserver
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.web
 import salt.netapi
 import salt.runner
 import salt.utils.args
@@ -211,7 +213,6 @@ from salt.exceptions import (
     AuthorizationError,
     EauthAuthenticationError,
 )
-from salt.ext.tornado.concurrent import Future
 from salt.utils.event import tagify
 
 _json = salt.utils.json.import_json()
@@ -277,7 +278,7 @@ class EventListener:
             opts["sock_dir"],
             opts=opts,
             listen=True,
-            io_loop=salt.ext.tornado.ioloop.IOLoop.current(),
+            io_loop=tornado.ioloop.IOLoop.current(),
         )
 
         # tag -> list of futures
@@ -302,9 +303,7 @@ class EventListener:
             self._timeout_future(tag, matcher, future)
             # remove the timeout
             if future in self.timeout_map:
-                salt.ext.tornado.ioloop.IOLoop.current().remove_timeout(
-                    self.timeout_map[future]
-                )
+                tornado.ioloop.IOLoop.current().remove_timeout(self.timeout_map[future])
                 del self.timeout_map[future]
 
         del self.request_map[request]
@@ -336,7 +335,7 @@ class EventListener:
         if callback is not None:
 
             def handle_future(future):
-                salt.ext.tornado.ioloop.IOLoop.current().add_callback(
+                tornado.ioloop.IOLoop.current().add_callback(
                     callback, future
                 )  # pylint: disable=E1102
 
@@ -346,7 +345,7 @@ class EventListener:
         self.request_map[request].append((tag, matcher, future))
 
         if timeout:
-            timeout_future = salt.ext.tornado.ioloop.IOLoop.current().call_later(
+            timeout_future = tornado.ioloop.IOLoop.current().call_later(
                 timeout, self._timeout_future, tag, matcher, future
             )
             self.timeout_map[future] = timeout_future
@@ -391,13 +390,13 @@ class EventListener:
                 future.set_result({"data": data, "tag": mtag})
                 self.tag_map[(tag, matcher)].remove(future)
                 if future in self.timeout_map:
-                    salt.ext.tornado.ioloop.IOLoop.current().remove_timeout(
+                    tornado.ioloop.IOLoop.current().remove_timeout(
                         self.timeout_map[future]
                     )
                     del self.timeout_map[future]
 
 
-class BaseSaltAPIHandler(salt.ext.tornado.web.RequestHandler):  # pylint: disable=W0223
+class BaseSaltAPIHandler(tornado.web.RequestHandler):  # pylint: disable=W0223
     ct_out_map = (
         ("application/json", _json_dumps),
         ("application/x-yaml", salt.utils.yaml.safe_dump),
@@ -559,7 +558,7 @@ class BaseSaltAPIHandler(salt.ext.tornado.web.RequestHandler):  # pylint: disabl
         try:
             # Use cgi.parse_header to correctly separate parameters from value
             value, parameters = cgi.parse_header(self.request.headers["Content-Type"])
-            return ct_in_map[value](salt.ext.tornado.escape.native_str(data))
+            return ct_in_map[value](tornado.escape.native_str(data))
         except KeyError:
             self.send_error(406)
         except ValueError:
@@ -836,7 +835,7 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
         self.write(self.serialize(ret))
         self.finish()
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def post(self):  # pylint: disable=arguments-differ
         """
         Send one or more Salt commands (lowstates) in the request body
@@ -914,7 +913,7 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
 
         self.disbatch()
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def disbatch(self):
         """
         Disbatch all lowstates to the appropriate clients
@@ -957,7 +956,7 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
         except RuntimeError as exc:
             log.exception("Encountered Runtime Error")
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def get_minion_returns(
         self, events, is_finished, is_timed_out, min_wait_time, minions
     ):
@@ -987,11 +986,11 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
                 # When finished entire routine, cleanup other futures and return result
                 if f is is_finished or f is is_timed_out:
                     cancel_inflight_futures()
-                    raise salt.ext.tornado.gen.Return(chunk_ret)
+                    raise tornado.gen.Return(chunk_ret)
                 elif f is min_wait_time:
                     if not more_todo():
                         cancel_inflight_futures()
-                        raise salt.ext.tornado.gen.Return(chunk_ret)
+                        raise tornado.gen.Return(chunk_ret)
                     continue
 
                 f_result = f.result()
@@ -1007,7 +1006,7 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
                     # if there are no more minions to wait for, then we are done
                     if not more_todo() and min_wait_time.done():
                         cancel_inflight_futures()
-                        raise salt.ext.tornado.gen.Return(chunk_ret)
+                        raise tornado.gen.Return(chunk_ret)
 
             except TimeoutException:
                 pass
@@ -1015,7 +1014,7 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
                 if f in events:
                     events.remove(f)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _disbatch_local(self, chunk):
         """
         Dispatch local client commands
@@ -1064,7 +1063,7 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
                     future.set_result(None)
                 except Exception:  # pylint: disable=broad-except
                     pass
-            raise salt.ext.tornado.gen.Return(
+            raise tornado.gen.Return(
                 "No minions matched the target. No command was sent, no jid was"
                 " assigned."
             )
@@ -1085,19 +1084,15 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
 
         # wait syndic a while to avoid missing published events
         if self.application.opts["order_masters"]:
-            min_wait_time = salt.ext.tornado.gen.sleep(
-                self.application.opts["syndic_wait"]
-            )
+            min_wait_time = tornado.gen.sleep(self.application.opts["syndic_wait"])
 
         # To ensure job_not_running and all_return are terminated by each other, communicate using a future
-        is_finished = salt.ext.tornado.gen.Future()
-        is_timed_out = salt.ext.tornado.gen.sleep(
-            self.application.opts["gather_job_timeout"]
-        )
+        is_finished = tornado.gen.Future()
+        is_timed_out = tornado.gen.sleep(self.application.opts["gather_job_timeout"])
 
         # ping until the job is not running, while doing so, if we see new minions returning
         # that they are running the job, add them to the list
-        salt.ext.tornado.ioloop.IOLoop.current().spawn_callback(
+        tornado.ioloop.IOLoop.current().spawn_callback(
             self.job_not_running,
             pub_data["jid"],
             chunk["tgt"],
@@ -1113,9 +1108,9 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
             min_wait_time=min_wait_time,
             minions=minions,
         )
-        raise salt.ext.tornado.gen.Return(result)
+        raise tornado.gen.Return(result)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def job_not_running(self, jid, tgt, tgt_type, minions, is_finished):
         """
         Return a future which will complete once jid (passed in) is no longer
@@ -1140,11 +1135,11 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
                 if f is is_finished:
                     if not event.done():
                         event.set_result(None)
-                    raise salt.ext.tornado.gen.Return(True)
+                    raise tornado.gen.Return(True)
                 event = f.result()
             except TimeoutException:
                 if not minion_running or is_finished.done():
-                    raise salt.ext.tornado.gen.Return(True)
+                    raise tornado.gen.Return(True)
                 else:
                     ping_pub_data = yield self.saltclients["local"](
                         tgt, "saltutil.find_job", [jid], tgt_type=tgt_type
@@ -1160,7 +1155,7 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
                 minions[event["data"]["id"]] = False
             minion_running = True
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _disbatch_local_async(self, chunk):
         """
         Disbatch local client_async commands
@@ -1171,9 +1166,9 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
             *f_call.get("args", ()), **f_call.get("kwargs", {})
         )
 
-        raise salt.ext.tornado.gen.Return(pub_data)
+        raise tornado.gen.Return(pub_data)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _disbatch_runner(self, chunk):
         """
         Disbatch runner client commands
@@ -1186,25 +1181,25 @@ class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
 
             # only return the return data
             ret = event if full_return else event["data"]["return"]
-            raise salt.ext.tornado.gen.Return(ret)
+            raise tornado.gen.Return(ret)
         except TimeoutException:
-            raise salt.ext.tornado.gen.Return("Timeout waiting for runner to execute")
+            raise tornado.gen.Return("Timeout waiting for runner to execute")
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _disbatch_runner_async(self, chunk):
         """
         Disbatch runner client_async commands
         """
         pub_data = self.saltclients["runner"](chunk)
-        raise salt.ext.tornado.gen.Return(pub_data)
+        raise tornado.gen.Return(pub_data)
 
     # salt.utils.args.format_call doesn't work for functions having the
-    # annotation salt.ext.tornado.gen.coroutine
+    # annotation tornado.gen.coroutine
     def _format_call_run_job_async(self, chunk):
         f_call = salt.utils.args.format_call(
             salt.client.LocalClient.run_job, chunk, is_class_method=True
         )
-        f_call.get("kwargs", {})["io_loop"] = salt.ext.tornado.ioloop.IOLoop.current()
+        f_call.get("kwargs", {})["io_loop"] = tornado.ioloop.IOLoop.current()
         return f_call
 
 
@@ -1213,7 +1208,7 @@ class MinionSaltAPIHandler(SaltAPIHandler):  # pylint: disable=W0223
     A convenience endpoint for minion related functions
     """
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def get(self, mid=None):  # pylint: disable=W0221
         """
         A convenience URL for getting lists of minions or getting minion
@@ -1261,7 +1256,7 @@ class MinionSaltAPIHandler(SaltAPIHandler):  # pylint: disable=W0223
         self.lowstate = [{"client": "local", "tgt": mid or "*", "fun": "grains.items"}]
         self.disbatch()
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def post(self):
         """
         Start an execution command and immediately return the job id
@@ -1339,7 +1334,7 @@ class JobsSaltAPIHandler(SaltAPIHandler):  # pylint: disable=W0223
     A convenience endpoint for job cache data
     """
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def get(self, jid=None):  # pylint: disable=W0221
         """
         A convenience URL for getting lists of previously run jobs or getting
@@ -1439,7 +1434,7 @@ class RunSaltAPIHandler(SaltAPIHandler):  # pylint: disable=W0223
     Endpoint to run commands without normal session handling
     """
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def post(self):
         """
         Run commands bypassing the :ref:`normal session handling
@@ -1511,7 +1506,7 @@ class EventsSaltAPIHandler(SaltAPIHandler):  # pylint: disable=W0223
     .. seealso:: :ref:`events`
     """
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def get(self):
         r"""
         An HTTP stream of the Salt master event bus
diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py
index e71db6cf99d..1e887a8f64c 100644
--- a/salt/netapi/rest_tornado/saltnado_websockets.py
+++ b/salt/netapi/rest_tornado/saltnado_websockets.py
@@ -291,8 +291,9 @@ Setup
 
 import logging
 
-import salt.ext.tornado.gen
-import salt.ext.tornado.websocket
+import tornado.gen
+import tornado.websocket
+
 import salt.netapi
 import salt.utils.json
 
@@ -306,7 +307,7 @@ log = logging.getLogger(__name__)
 
 
 class AllEventsHandler(
-    salt.ext.tornado.websocket.WebSocketHandler
+    tornado.websocket.WebSocketHandler
 ):  # pylint: disable=W0223,W0232
     """
     Server side websocket handler.
@@ -334,7 +335,7 @@ class AllEventsHandler(
         """
         self.connected = False
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def on_message(self, message):
         """Listens for a "websocket client ready" message.
         Once that message is received an asynchronous job
@@ -386,7 +387,7 @@ class AllEventsHandler(
 
 
 class FormattedEventsHandler(AllEventsHandler):  # pylint: disable=W0223,W0232
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def on_message(self, message):
         """Listens for a "websocket client ready" message.
         Once that message is received an asynchronous job
diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py
index d324fcee192..bb39be58f2a 100644
--- a/salt/pillar/__init__.py
+++ b/salt/pillar/__init__.py
@@ -10,8 +10,9 @@ import os
 import sys
 import traceback
 
+import tornado.gen
+
 import salt.channel.client
-import salt.ext.tornado.gen
 import salt.fileclient
 import salt.loader
 import salt.minion
@@ -239,7 +240,7 @@ class AsyncRemotePillar(RemotePillarMixin):
         self._closing = False
         self.clean_cache = clean_cache
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def compile_pillar(self):
         """
         Return a future which will contain the pillar data from the master
@@ -277,7 +278,7 @@ class AsyncRemotePillar(RemotePillarMixin):
             log.error(msg)
             # raise an exception! Pillar isn't empty, we can't sync it!
             raise SaltClientError(msg)
-        raise salt.ext.tornado.gen.Return(ret_pillar)
+        raise tornado.gen.Return(ret_pillar)
 
     def destroy(self):
         if self._closing:
@@ -1356,7 +1357,7 @@ class Pillar:
 # TODO: actually migrate from Pillar to AsyncPillar to allow for futures in
 # ext_pillar etc.
 class AsyncPillar(Pillar):
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def compile_pillar(self, ext=True):
         ret = super().compile_pillar(ext=ext)
-        raise salt.ext.tornado.gen.Return(ret)
+        raise tornado.gen.Return(ret)
diff --git a/salt/transport/base.py b/salt/transport/base.py
index 014a9731d59..86a2c882deb 100644
--- a/salt/transport/base.py
+++ b/salt/transport/base.py
@@ -1,4 +1,4 @@
-import salt.ext.tornado.gen
+import tornado.gen
 
 TRANSPORTS = (
     "zeromq",
@@ -103,7 +103,7 @@ class RequestClient:
     def __init__(self, opts, io_loop, **kwargs):
         pass
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def send(self, load, timeout=60):
         """
         Send a request message and return the reply from the server.
@@ -211,7 +211,7 @@ class PublishClient:
         """
         raise NotImplementedError
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def connect(self, publish_port, connect_callback=None, disconnect_callback=None):
         """
         Create a network connection to the the PublishServer or broker.
diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py
index 453afaaad78..f80dd0e7562 100644
--- a/salt/transport/ipc.py
+++ b/salt/transport/ipc.py
@@ -8,17 +8,18 @@ import logging
 import socket
 import time
 
-import salt.ext.tornado
-import salt.ext.tornado.concurrent
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.netutil
+import tornado
+import tornado.concurrent
+import tornado.gen
+import tornado.ioloop
+import tornado.netutil
+from tornado.ioloop import IOLoop
+from tornado.ioloop import TimeoutError as TornadoTimeoutError
+from tornado.iostream import IOStream, StreamClosedError
+from tornado.locks import Lock
+
 import salt.transport.frame
 import salt.utils.msgpack
-from salt.ext.tornado.ioloop import IOLoop
-from salt.ext.tornado.ioloop import TimeoutError as TornadoTimeoutError
-from salt.ext.tornado.iostream import IOStream, StreamClosedError
-from salt.ext.tornado.locks import Lock
 
 log = logging.getLogger(__name__)
 
@@ -32,7 +33,7 @@ def future_with_timeout_callback(future):
         future._future_with_timeout._done_callback(future)
 
 
-class FutureWithTimeout(salt.ext.tornado.concurrent.Future):
+class FutureWithTimeout(tornado.concurrent.Future):
     def __init__(self, io_loop, future, timeout):
         super().__init__()
         self.io_loop = io_loop
@@ -112,7 +113,7 @@ class IPCServer:
 
         # Placeholders for attributes to be populated by method calls
         self.sock = None
-        self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current()
+        self.io_loop = io_loop or tornado.ioloop.IOLoop.current()
         self._closing = False
 
     def start(self):
@@ -131,16 +132,16 @@ class IPCServer:
             # Based on default used in tornado.netutil.bind_sockets()
             self.sock.listen(128)
         else:
-            self.sock = salt.ext.tornado.netutil.bind_unix_socket(self.socket_path)
+            self.sock = tornado.netutil.bind_unix_socket(self.socket_path)
 
         with salt.utils.asynchronous.current_ioloop(self.io_loop):
-            salt.ext.tornado.netutil.add_accept_handler(
+            tornado.netutil.add_accept_handler(
                 self.sock,
                 self.handle_connection,
             )
         self._started = True
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def handle_stream(self, stream):
         """
         Override this to handle the streams as they arrive
@@ -151,14 +152,14 @@ class IPCServer:
         for additional details.
         """
 
-        @salt.ext.tornado.gen.coroutine
+        @tornado.gen.coroutine
         def _null(msg):
-            raise salt.ext.tornado.gen.Return(None)
+            raise tornado.gen.Return(None)
 
         def write_callback(stream, header):
             if header.get("mid"):
 
-                @salt.ext.tornado.gen.coroutine
+                @tornado.gen.coroutine
                 def return_message(msg):
                     pack = salt.transport.frame.frame_msg_ipc(
                         msg,
@@ -276,7 +277,7 @@ class IPCClient:
         to the server.
 
         """
-        self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current()
+        self.io_loop = io_loop or tornado.ioloop.IOLoop.current()
         self.socket_path = socket_path
         self._closing = False
         self.stream = None
@@ -302,7 +303,7 @@ class IPCClient:
             if self._connecting_future is not None:
                 # read previous future result to prevent the "unhandled future exception" error
                 self._connecting_future.exception()  # pylint: disable=E0203
-            future = salt.ext.tornado.concurrent.Future()
+            future = tornado.concurrent.Future()
             self._connecting_future = future
             self._connect(timeout)
 
@@ -316,7 +317,7 @@ class IPCClient:
 
         return future
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _connect(self, timeout=None):
         """
         Connect to a running IPCServer
@@ -355,7 +356,7 @@ class IPCClient:
                     self._connecting_future.set_exception(e)
                     break
 
-                yield salt.ext.tornado.gen.sleep(1)
+                yield tornado.gen.sleep(1)
 
     def close(self):
         """
@@ -409,13 +410,13 @@ class IPCMessageClient(IPCClient):
     IMPORTANT: The below example also assumes a running IOLoop process.
 
     # Import Tornado libs
-    import salt.ext.tornado.ioloop
+    import tornado.ioloop
 
     # Import Salt libs
     import salt.config
     import salt.transport.ipc
 
-    io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+    io_loop = tornado.ioloop.IOLoop.current()
 
     ipc_server_socket_path = '/var/run/ipc_server.ipc'
 
@@ -439,7 +440,7 @@ class IPCMessageClient(IPCClient):
 
     # FIXME timeout unimplemented
     # FIXME tries unimplemented
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def send(self, msg, timeout=None, tries=None):
         """
         Send a message to an IPC socket
@@ -466,12 +467,12 @@ class IPCMessageServer(IPCServer):
     a console:
 
         # Import Tornado libs
-        import salt.ext.tornado.ioloop
+        import tornado.ioloop
 
         # Import Salt libs
         import salt.transport.ipc
 
-        io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+        io_loop = tornado.ioloop.IOLoop.current()
         ipc_server_socket_path = '/var/run/ipc_server.ipc'
         ipc_server = salt.transport.ipc.IPCMessageServer(ipc_server_socket_path, io_loop=io_loop,
                                                          payload_handler=print_to_console)
@@ -532,19 +533,19 @@ class IPCMessagePublisher:
             self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
             self.sock.setblocking(0)
             self.sock.bind(("127.0.0.1", self.socket_path))
-            # Based on default used in salt.ext.tornado.netutil.bind_sockets()
+            # Based on default used in tornado.netutil.bind_sockets()
             self.sock.listen(128)
         else:
-            self.sock = salt.ext.tornado.netutil.bind_unix_socket(self.socket_path)
+            self.sock = tornado.netutil.bind_unix_socket(self.socket_path)
 
         with salt.utils.asynchronous.current_ioloop(self.io_loop):
-            salt.ext.tornado.netutil.add_accept_handler(
+            tornado.netutil.add_accept_handler(
                 self.sock,
                 self.handle_connection,
             )
         self._started = True
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _write(self, stream, pack):
         try:
             yield stream.write(pack)
@@ -623,7 +624,7 @@ class IPCMessageSubscriber(IPCClient):
     IMPORTANT: The below example also assumes the IOLoop is NOT running.
 
     # Import Tornado libs
-    import salt.ext.tornado.ioloop
+    import tornado.ioloop
 
     # Import Salt libs
     import salt.config
@@ -631,7 +632,7 @@ class IPCMessageSubscriber(IPCClient):
 
     # Create a new IO Loop.
     # We know that this new IO Loop is not currently running.
-    io_loop = salt.ext.tornado.ioloop.IOLoop()
+    io_loop = tornado.ioloop.IOLoop()
 
     ipc_publisher_socket_path = '/var/run/ipc_publisher.ipc'
 
@@ -659,13 +660,13 @@ class IPCMessageSubscriber(IPCClient):
         self._saved_data = []
         self._read_in_progress = Lock()
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _read(self, timeout, callback=None):
         try:
             try:
                 yield self._read_in_progress.acquire(timeout=0.00000001)
-            except salt.ext.tornado.gen.TimeoutError:
-                raise salt.ext.tornado.gen.Return(None)
+            except tornado.gen.TimeoutError:
+                raise tornado.gen.Return(None)
 
             exc_to_raise = None
             ret = None
@@ -720,12 +721,12 @@ class IPCMessageSubscriber(IPCClient):
 
             if exc_to_raise is not None:
                 raise exc_to_raise  # pylint: disable=E0702
-            raise salt.ext.tornado.gen.Return(ret)
+            raise tornado.gen.Return(ret)
         # Handle ctrl+c gracefully
         except TypeError:
             pass
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def read(self, timeout):
         """
         Asynchronously read messages and invoke a callback when they are ready.
@@ -733,7 +734,7 @@ class IPCMessageSubscriber(IPCClient):
         """
         if self._saved_data:
             res = self._saved_data.pop(0)
-            raise salt.ext.tornado.gen.Return(res)
+            raise tornado.gen.Return(res)
         while not self.connected():
             try:
                 yield self.connect(timeout=5)
@@ -742,12 +743,12 @@ class IPCMessageSubscriber(IPCClient):
                     "Subscriber closed stream on IPC %s before connect",
                     self.socket_path,
                 )
-                yield salt.ext.tornado.gen.sleep(1)
+                yield tornado.gen.sleep(1)
             except Exception as exc:  # pylint: disable=broad-except
                 log.error("Exception occurred while Subscriber connecting: %s", exc)
-                yield salt.ext.tornado.gen.sleep(1)
+                yield tornado.gen.sleep(1)
         res = yield self._read(timeout)
-        raise salt.ext.tornado.gen.Return(res)
+        raise tornado.gen.Return(res)
 
     def read_sync(self, timeout=None):
         """
@@ -763,7 +764,7 @@ class IPCMessageSubscriber(IPCClient):
             return self._saved_data.pop(0)
         return self.io_loop.run_sync(lambda: self._read(timeout))
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def read_async(self, callback):
         """
         Asynchronously read messages and invoke a callback when they are ready.
@@ -778,10 +779,10 @@ class IPCMessageSubscriber(IPCClient):
                     "Subscriber closed stream on IPC %s before connect",
                     self.socket_path,
                 )
-                yield salt.ext.tornado.gen.sleep(1)
+                yield tornado.gen.sleep(1)
             except Exception as exc:  # pylint: disable=broad-except
                 log.error("Exception occurred while Subscriber connecting: %s", exc)
-                yield salt.ext.tornado.gen.sleep(1)
+                yield tornado.gen.sleep(1)
         yield self._read(None, callback)
 
     def close(self):
diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py
index a28ac19338e..9c868d6e7cf 100644
--- a/salt/transport/tcp.py
+++ b/salt/transport/tcp.py
@@ -16,13 +16,14 @@ import socket
 import threading
 import urllib
 
-import salt.ext.tornado
-import salt.ext.tornado.concurrent
-import salt.ext.tornado.gen
-import salt.ext.tornado.iostream
-import salt.ext.tornado.netutil
-import salt.ext.tornado.tcpclient
-import salt.ext.tornado.tcpserver
+import tornado
+import tornado.concurrent
+import tornado.gen
+import tornado.iostream
+import tornado.netutil
+import tornado.tcpclient
+import tornado.tcpserver
+
 import salt.master
 import salt.payload
 import salt.transport.frame
@@ -41,7 +42,8 @@ else:
     USE_LOAD_BALANCER = False
 
 if USE_LOAD_BALANCER:
-    import salt.ext.tornado.util
+    import tornado.util
+
     from salt.utils.process import SignalHandlingProcess
 
 log = logging.getLogger(__name__)
@@ -139,7 +141,7 @@ if USE_LOAD_BALANCER:
         """
 
         # TODO: opts!
-        # Based on default used in salt.ext.tornado.netutil.bind_sockets()
+        # Based on default used in tornado.netutil.bind_sockets()
         backlog = 128
 
         def __init__(self, opts, socket_queue, **kwargs):
@@ -184,10 +186,7 @@ if USE_LOAD_BALANCER:
                     # ECONNABORTED indicates that there was a connection
                     # but it was closed while still in the accept queue.
                     # (observed on FreeBSD).
-                    if (
-                        salt.ext.tornado.util.errno_from_exception(e)
-                        == errno.ECONNABORTED
-                    ):
+                    if tornado.util.errno_from_exception(e) == errno.ECONNABORTED:
                         continue
                     raise
 
@@ -198,8 +197,8 @@ class Resolver:
 
     @classmethod
     def _config_resolver(cls, num_threads=10):
-        salt.ext.tornado.netutil.Resolver.configure(
-            "salt.ext.tornado.netutil.ThreadedResolver", num_threads=num_threads
+        tornado.netutil.Resolver.configure(
+            "tornado.netutil.ThreadedResolver", num_threads=num_threads
         )
         cls._resolver_configured = True
 
@@ -238,7 +237,7 @@ class TCPPubClient(salt.transport.base.PublishClient):
 
     # pylint: enable=W1701
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def connect(self, publish_port, connect_callback=None, disconnect_callback=None):
         self.publish_port = publish_port
         self.message_client = MessageClient(
@@ -254,7 +253,7 @@ class TCPPubClient(salt.transport.base.PublishClient):
         yield self.message_client.connect()  # wait for the client to be connected
         self.connected = True
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _decode_messages(self, messages):
         if not isinstance(messages, dict):
             # TODO: For some reason we need to decode here for things
@@ -263,9 +262,9 @@ class TCPPubClient(salt.transport.base.PublishClient):
             body = salt.transport.frame.decode_embedded_strs(body)
         else:
             body = messages
-        raise salt.ext.tornado.gen.Return(body)
+        raise tornado.gen.Return(body)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def send(self, msg):
         yield self.message_client._stream.write(msg)
 
@@ -387,7 +386,7 @@ class TCPReqServer(salt.transport.base.DaemonizedRequestServer):
                 self.req_server.add_socket(self._socket)
                 self._socket.listen(self.backlog)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def handle_message(self, stream, payload, header=None):
         payload = self.decode_payload(payload)
         reply = yield self.message_handler(payload)
@@ -397,28 +396,26 @@ class TCPReqServer(salt.transport.base.DaemonizedRequestServer):
         return payload
 
 
-class SaltMessageServer(salt.ext.tornado.tcpserver.TCPServer):
+class SaltMessageServer(tornado.tcpserver.TCPServer):
     """
     Raw TCP server which will receive all of the TCP streams and re-assemble
     messages that are sent through to us
     """
 
     def __init__(self, message_handler, *args, **kwargs):
-        io_loop = (
-            kwargs.pop("io_loop", None) or salt.ext.tornado.ioloop.IOLoop.current()
-        )
+        io_loop = kwargs.pop("io_loop", None) or tornado.ioloop.IOLoop.current()
         self._closing = False
         super().__init__(*args, **kwargs)
         self.io_loop = io_loop
         self.clients = []
         self.message_handler = message_handler
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def handle_stream(  # pylint: disable=arguments-differ
         self,
         stream,
         address,
-        _StreamClosedError=salt.ext.tornado.iostream.StreamClosedError,
+        _StreamClosedError=tornado.iostream.StreamClosedError,
     ):
         """
         Handle incoming streams and add messages to the incoming queue
@@ -500,7 +497,7 @@ if USE_LOAD_BALANCER:
                             break
                         continue
                     # 'self.io_loop' initialized in super class
-                    # 'salt.ext.tornado.tcpserver.TCPServer'.
+                    # 'tornado.tcpserver.TCPServer'.
                     # 'self._handle_connection' defined in same super class.
                     self.io_loop.spawn_callback(
                         self._handle_connection, client_socket, address
@@ -509,7 +506,7 @@ if USE_LOAD_BALANCER:
                 pass
 
 
-class TCPClientKeepAlive(salt.ext.tornado.tcpclient.TCPClient):
+class TCPClientKeepAlive(tornado.tcpclient.TCPClient):
     """
     Override _create_stream() in TCPClient to enable keep alive support.
     """
@@ -532,11 +529,7 @@ class TCPClientKeepAlive(salt.ext.tornado.tcpclient.TCPClient):
         # after one connection has completed.
         sock = _get_socket(self.opts)
         _set_tcp_keepalive(sock, self.opts)
-        stream = salt.ext.tornado.iostream.IOStream(
-            sock, max_buffer_size=max_buffer_size
-        )
-        if salt.ext.tornado.version_info < (5,):
-            return stream.connect(addr)
+        stream = tornado.iostream.IOStream(sock, max_buffer_size=max_buffer_size)
         return stream, stream.connect(addr)
 
 
@@ -567,7 +560,7 @@ class MessageClient:
         self.source_port = source_port
         self.connect_callback = connect_callback
         self.disconnect_callback = disconnect_callback
-        self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current()
+        self.io_loop = io_loop or tornado.ioloop.IOLoop.current()
         with salt.utils.asynchronous.current_ioloop(self.io_loop):
             self._tcp_client = TCPClientKeepAlive(opts, resolver=resolver)
         self._mid = 1
@@ -580,7 +573,7 @@ class MessageClient:
         self._on_recv = None
         self._closing = False
         self._closed = False
-        self._connecting_future = salt.ext.tornado.concurrent.Future()
+        self._connecting_future = tornado.concurrent.Future()
         self._stream_return_running = False
         self._stream = None
 
@@ -597,7 +590,7 @@ class MessageClient:
         self._closing = True
         self.io_loop.add_timeout(1, self.check_close)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def check_close(self):
         if not self.send_future_map:
             self._tcp_client.close()
@@ -613,7 +606,7 @@ class MessageClient:
 
     # pylint: enable=W1701
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def getstream(self, **kwargs):
         if self.source_ip or self.source_port:
             kwargs = {
@@ -638,10 +631,10 @@ class MessageClient:
                     exc,
                     self.backoff,
                 )
-                yield salt.ext.tornado.gen.sleep(self.backoff)
-        raise salt.ext.tornado.gen.Return(stream)
+                yield tornado.gen.sleep(self.backoff)
+        raise tornado.gen.Return(stream)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def connect(self):
         if self._stream is None:
             self._stream = yield self.getstream()
@@ -651,7 +644,7 @@ class MessageClient:
                 if self.connect_callback:
                     self.connect_callback(True)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _stream_return(self):
         self._stream_return_running = True
         unpacker = salt.utils.msgpack.Unpacker()
@@ -677,7 +670,7 @@ class MessageClient:
                                 " tracking",
                                 message_id,
                             )
-            except salt.ext.tornado.iostream.StreamClosedError as e:
+            except tornado.iostream.StreamClosedError as e:
                 log.debug(
                     "tcp stream to %s:%s closed, unable to recv",
                     self.host,
@@ -763,14 +756,14 @@ class MessageClient:
         if future is not None:
             future.set_exception(SaltReqTimeoutError("Message timed out"))
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def send(self, msg, timeout=None, callback=None, raw=False):
         if self._closing:
             raise ClosingError()
         message_id = self._message_id()
         header = {"mid": message_id}
 
-        future = salt.ext.tornado.concurrent.Future()
+        future = tornado.concurrent.Future()
 
         if callback is not None:
 
@@ -790,7 +783,7 @@ class MessageClient:
 
         item = salt.transport.frame.frame_msg(msg, header=header)
 
-        @salt.ext.tornado.gen.coroutine
+        @tornado.gen.coroutine
         def _do_send():
             yield self.connect()
             # If the _stream is None, we failed to connect.
@@ -801,7 +794,7 @@ class MessageClient:
         # out before we are able to connect.
         self.io_loop.add_callback(_do_send)
         recv = yield future
-        raise salt.ext.tornado.gen.Return(recv)
+        raise tornado.gen.Return(recv)
 
 
 class Subscriber:
@@ -838,7 +831,7 @@ class Subscriber:
     # pylint: enable=W1701
 
 
-class PubServer(salt.ext.tornado.tcpserver.TCPServer):
+class PubServer(tornado.tcpserver.TCPServer):
     """
     TCP publisher
     """
@@ -874,7 +867,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer):
 
     # pylint: enable=W1701
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _stream_read(self, client):
         unpacker = salt.utils.msgpack.Unpacker()
         while not self._closing:
@@ -887,7 +880,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer):
                     body = framed_msg["body"]
                     if self.presence_callback:
                         self.presence_callback(client, body)
-            except salt.ext.tornado.iostream.StreamClosedError as e:
+            except tornado.iostream.StreamClosedError as e:
                 log.debug("tcp stream to %s closed, unable to recv", client.address)
                 client.close()
                 self.remove_presence_callback(client)
@@ -906,7 +899,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer):
         self.io_loop.spawn_callback(self._stream_read, client)
 
     # TODO: ACK the publish through IPC
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def publish_payload(self, package, topic_list=None):
         log.trace("TCP PubServer sending payload: %s \n\n %r", package, topic_list)
         payload = salt.transport.frame.frame_msg(package)
@@ -921,7 +914,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer):
                             yield client.stream.write(payload)
                             sent = True
                             # self.io_loop.add_future(f, lambda f: True)
-                        except salt.ext.tornado.iostream.StreamClosedError:
+                        except tornado.iostream.StreamClosedError:
                             to_remove.append(client)
                 if not sent:
                     log.debug("Publish target %s not connected %r", topic, self.clients)
@@ -930,7 +923,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer):
                 try:
                     # Write the packed str
                     yield client.stream.write(payload)
-                except salt.ext.tornado.iostream.StreamClosedError:
+                except tornado.iostream.StreamClosedError:
                     to_remove.append(client)
         for client in to_remove:
             log.debug(
@@ -948,7 +941,7 @@ class TCPPublishServer(salt.transport.base.DaemonizedPublishServer):
     """
 
     # TODO: opts!
-    # Based on default used in salt.ext.tornado.netutil.bind_sockets()
+    # Based on default used in tornado.netutil.bind_sockets()
     backlog = 128
 
     def __init__(self, opts):
@@ -974,7 +967,7 @@ class TCPPublishServer(salt.transport.base.DaemonizedPublishServer):
         """
         Bind to the interface specified in the configuration file
         """
-        io_loop = salt.ext.tornado.ioloop.IOLoop()
+        io_loop = tornado.ioloop.IOLoop()
         io_loop.make_current()
 
         # Spin up the publisher
@@ -1026,10 +1019,10 @@ class TCPPublishServer(salt.transport.base.DaemonizedPublishServer):
         """
         process_manager.add_process(self.publish_daemon, name=self.__class__.__name__)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def publish_payload(self, payload, *args):
         ret = yield self.pub_server.publish_payload(payload, *args)
-        raise salt.ext.tornado.gen.Return(ret)
+        raise tornado.gen.Return(ret)
 
     def publish(self, payload, **kwargs):
         """
@@ -1079,14 +1072,14 @@ class TCPReqClient(salt.transport.base.RequestClient):
             source_port=opts.get("source_ret_port"),
         )
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def connect(self):
         yield self.message_client.connect()
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def send(self, load, timeout=60):
         ret = yield self.message_client.send(load, timeout=timeout)
-        raise salt.ext.tornado.gen.Return(ret)
+        raise tornado.gen.Return(ret)
 
     def close(self):
         self.message_client.close()
diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py
index 3ec7f7726c4..5effe0d6fa8 100644
--- a/salt/transport/zeromq.py
+++ b/salt/transport/zeromq.py
@@ -10,13 +10,13 @@ import sys
 import threading
 from random import randint
 
+import tornado
+import tornado.concurrent
+import tornado.gen
+import tornado.ioloop
 import zmq.error
 import zmq.eventloop.zmqstream
 
-import salt.ext.tornado
-import salt.ext.tornado.concurrent
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
 import salt.payload
 import salt.transport.base
 import salt.utils.files
@@ -203,7 +203,7 @@ class PublishClient(salt.transport.base.PublishClient):
         self.close()
 
     # TODO: this is the time to see if we are connected, maybe use the req channel to guess?
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def connect(self, publish_port, connect_callback=None, disconnect_callback=None):
         self.publish_port = publish_port
         log.debug(
@@ -226,7 +226,7 @@ class PublishClient(salt.transport.base.PublishClient):
             source_port=self.opts.get("source_publish_port"),
         )
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _decode_messages(self, messages):
         """
         Take the zmq messages, decrypt/decode them into a payload
@@ -248,7 +248,7 @@ class PublishClient(salt.transport.base.PublishClient):
                 and message_target not in ("broadcast", "syndic")
             ):
                 log.debug("Publish received for not this minion: %s", message_target)
-                raise salt.ext.tornado.gen.Return(None)
+                raise tornado.gen.Return(None)
             payload = salt.payload.loads(messages[1])
         else:
             raise Exception(
@@ -258,7 +258,7 @@ class PublishClient(salt.transport.base.PublishClient):
             )
         # Yield control back to the caller. When the payload has been decoded, assign
         # the decoded payload to 'ret' and resume operation
-        raise salt.ext.tornado.gen.Return(payload)
+        raise tornado.gen.Return(payload)
 
     @property
     def stream(self):
@@ -279,7 +279,7 @@ class PublishClient(salt.transport.base.PublishClient):
         """
         return self.stream.on_recv(callback)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def send(self, msg):
         self.stream.send(msg, noblock=True)
 
@@ -426,7 +426,7 @@ class RequestServer(salt.transport.base.DaemonizedRequestServer):
         self.message_handler = message_handler
         self.stream.on_recv_stream(self.handle_message)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def handle_message(self, stream, payload):
         payload = self.decode_payload(payload)
         # XXX: Is header really needed?
@@ -504,7 +504,7 @@ class AsyncReqMessageClient:
         self.addr = addr
         self.linger = linger
         if io_loop is None:
-            self.io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+            self.io_loop = tornado.ioloop.IOLoop.current()
         else:
             self.io_loop = io_loop
 
@@ -587,12 +587,12 @@ class AsyncReqMessageClient:
         if future is not None:
             future.set_exception(SaltReqTimeoutError("Message timed out"))
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def send(self, message, timeout=None, callback=None):
         """
         Return a future which will be completed when the message has a response
         """
-        future = salt.ext.tornado.concurrent.Future()
+        future = tornado.concurrent.Future()
 
         message = salt.payload.dumps(message)
 
@@ -624,7 +624,7 @@ class AsyncReqMessageClient:
         self.stream.on_recv(mark_future)
         yield self.stream.send(message)
         recv = yield future
-        raise salt.ext.tornado.gen.Return(recv)
+        raise tornado.gen.Return(recv)
 
 
 class ZeroMQSocketMonitor:
@@ -700,7 +700,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer):
         self.opts = opts
 
     def connect(self):
-        return salt.ext.tornado.gen.sleep(5)
+        return tornado.gen.sleep(5)
 
     def publish_daemon(
         self,
@@ -712,7 +712,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer):
         This method represents the Publish Daemon process. It is intended to be
         run in a thread or process as it creates and runs an it's own ioloop.
         """
-        ioloop = salt.ext.tornado.ioloop.IOLoop()
+        ioloop = tornado.ioloop.IOLoop()
         ioloop.make_current()
         self.io_loop = ioloop
         context = zmq.Context(1)
@@ -748,7 +748,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer):
         with salt.utils.files.set_umask(0o177):
             pull_sock.bind(self.pull_uri)
 
-        @salt.ext.tornado.gen.coroutine
+        @tornado.gen.coroutine
         def on_recv(packages):
             for package in packages:
                 payload = salt.payload.loads(package)
@@ -777,7 +777,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer):
     def pub_uri(self):
         return "tcp://{interface}:{publish_port}".format(**self.opts)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def publish_payload(self, payload, topic_list=None):
         payload = salt.payload.dumps(payload)
         if self.opts["zmq_filtering"]:
@@ -910,11 +910,11 @@ class RequestClient(salt.transport.base.RequestClient):
     def connect(self):
         self.message_client.connect()
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def send(self, load, timeout=60):
         self.connect()
         ret = yield self.message_client.send(load, timeout=timeout)
-        raise salt.ext.tornado.gen.Return(ret)
+        raise tornado.gen.Return(ret)
 
     def close(self):
         self.message_client.close()
diff --git a/salt/utils/asynchronous.py b/salt/utils/asynchronous.py
index f081e43e6f1..e887cb693f0 100644
--- a/salt/utils/asynchronous.py
+++ b/salt/utils/asynchronous.py
@@ -8,8 +8,8 @@ import logging
 import sys
 import threading
 
-import salt.ext.tornado.concurrent
-import salt.ext.tornado.ioloop
+import tornado.concurrent
+import tornado.ioloop
 
 log = logging.getLogger(__name__)
 
@@ -20,7 +20,7 @@ def current_ioloop(io_loop):
     A context manager that will set the current ioloop to io_loop for the context
     """
     try:
-        orig_loop = salt.ext.tornado.ioloop.IOLoop.current()
+        orig_loop = tornado.ioloop.IOLoop.current()
     except RuntimeError:
         orig_loop = None
     io_loop.make_current()
@@ -57,7 +57,7 @@ class SyncWrapper:
         close_methods=None,
         loop_kwarg=None,
     ):
-        self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+        self.io_loop = tornado.ioloop.IOLoop()
         if args is None:
             args = []
         if kwargs is None:
diff --git a/salt/utils/event.py b/salt/utils/event.py
index a07ad513b1c..c91f17fda75 100644
--- a/salt/utils/event.py
+++ b/salt/utils/event.py
@@ -60,11 +60,12 @@ import os
 import time
 from collections.abc import MutableMapping
 
+import tornado.ioloop
+import tornado.iostream
+
 import salt.channel.client
 import salt.config
 import salt.defaults.exitcodes
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.iostream
 import salt.payload
 import salt.transport.ipc
 import salt.utils.asynchronous
@@ -236,7 +237,7 @@ class SaltEvent:
             self.io_loop = io_loop
             self._run_io_loop_sync = False
         else:
-            self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+            self.io_loop = tornado.ioloop.IOLoop()
             self._run_io_loop_sync = True
         self.cpub = False
         self.cpush = False
@@ -371,7 +372,7 @@ class SaltEvent:
                 try:
                     self.subscriber.connect(timeout=timeout)
                     self.cpub = True
-                except salt.ext.tornado.iostream.StreamClosedError:
+                except tornado.iostream.StreamClosedError:
                     log.error("Encountered StreamClosedException")
                 except OSError as exc:
                     if exc.errno != errno.ENOENT:
@@ -426,7 +427,7 @@ class SaltEvent:
                 try:
                     self.pusher.connect(timeout=timeout)
                     self.cpush = True
-                except salt.ext.tornado.iostream.StreamClosedError as exc:
+                except tornado.iostream.StreamClosedError as exc:
                     log.debug("Unable to connect pusher: %s", exc)
                 except Exception as exc:  # pylint: disable=broad-except
                     log.error(
@@ -578,7 +579,7 @@ class SaltEvent:
                 ret = {"data": data, "tag": mtag}
             except KeyboardInterrupt:
                 return {"tag": "salt/event/exit", "data": {}}
-            except salt.ext.tornado.iostream.StreamClosedError:
+            except tornado.iostream.StreamClosedError:
                 if self.raise_errors:
                     raise
                 else:
@@ -672,7 +673,7 @@ class SaltEvent:
                         try:
                             ret = self._get_event(wait, tag, match_func, no_block)
                             break
-                        except salt.ext.tornado.iostream.StreamClosedError:
+                        except tornado.iostream.StreamClosedError:
                             self.close_pub()
                             self.connect_pub(timeout=wait)
                             continue
@@ -728,7 +729,7 @@ class SaltEvent:
                 continue
             yield data
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def fire_event_async(self, data, tag, cb=None, timeout=1000):
         """
         Send a single event into the publisher with payload dict "data" and
@@ -1066,7 +1067,7 @@ class AsyncEventPublisher:
         default_minion_sock_dir = self.opts["sock_dir"]
         self.opts.update(opts)
 
-        self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current()
+        self.io_loop = io_loop or tornado.ioloop.IOLoop.current()
         self._closing = False
         self.publisher = None
         self.puller = None
@@ -1184,7 +1185,7 @@ class EventPublisher(salt.utils.process.SignalHandlingProcess):
             )
             os.nice(self.opts["event_publisher_niceness"])
 
-        self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+        self.io_loop = tornado.ioloop.IOLoop()
         with salt.utils.asynchronous.current_ioloop(self.io_loop):
             if self.opts["ipc_mode"] == "tcp":
                 epub_uri = int(self.opts["tcp_master_pub_port"])
diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py
index cc9895d8ab9..5bacfdcc55f 100644
--- a/salt/utils/gitfs.py
+++ b/salt/utils/gitfs.py
@@ -20,7 +20,8 @@ import time
 import weakref
 from datetime import datetime
 
-import salt.ext.tornado.ioloop
+import tornado.ioloop
+
 import salt.fileserver
 import salt.utils.configparser
 import salt.utils.data
@@ -2858,7 +2859,7 @@ class GitFS(GitBase):
         exited.
         """
         # No need to get the ioloop reference if we're not initializing remotes
-        io_loop = salt.ext.tornado.ioloop.IOLoop.current() if init_remotes else None
+        io_loop = tornado.ioloop.IOLoop.current() if init_remotes else None
         if not init_remotes or io_loop not in cls.instance_map:
             # We only evaluate the second condition in this if statement if
             # we're initializing remotes, so we won't get here unless io_loop
diff --git a/salt/utils/http.py b/salt/utils/http.py
index 91c5cbf08ed..9928847ed62 100644
--- a/salt/utils/http.py
+++ b/salt/utils/http.py
@@ -22,9 +22,11 @@ import urllib.request
 import xml.etree.ElementTree as ET
 import zlib
 
+import tornado.httputil
+import tornado.simple_httpclient
+from tornado.httpclient import HTTPClient
+
 import salt.config
-import salt.ext.tornado.httputil
-import salt.ext.tornado.simple_httpclient
 import salt.loader
 import salt.syspaths
 import salt.utils.args
@@ -38,7 +40,6 @@ import salt.utils.stringutils
 import salt.utils.xmlutil as xml
 import salt.utils.yaml
 import salt.version
-from salt.ext.tornado.httpclient import HTTPClient
 from salt.template import compile_template
 from salt.utils.decorators.jinja import jinja_filter
 
@@ -63,7 +64,7 @@ except ImportError:
 
 
 try:
-    import salt.ext.tornado.curl_httpclient
+    import tornado.curl_httpclient
 
     HAS_CURL_HTTPCLIENT = True
 except ImportError:
@@ -213,7 +214,7 @@ def query(
 
     # Some libraries don't support separation of url and GET parameters
     # Don't need a try/except block, since Salt depends on tornado
-    url_full = salt.ext.tornado.httputil.url_concat(url, params) if params else url
+    url_full = tornado.httputil.url_concat(url, params) if params else url
 
     if ca_bundle is None:
         ca_bundle = get_ca_bundle(opts)
@@ -567,16 +568,16 @@ def query(
                 log.error(ret["error"])
                 return ret
 
-            salt.ext.tornado.httpclient.AsyncHTTPClient.configure(
+            tornado.httpclient.AsyncHTTPClient.configure(
                 "tornado.curl_httpclient.CurlAsyncHTTPClient"
             )
             client_argspec = salt.utils.args.get_function_argspec(
-                salt.ext.tornado.curl_httpclient.CurlAsyncHTTPClient.initialize
+                tornado.curl_httpclient.CurlAsyncHTTPClient.initialize
             )
         else:
-            salt.ext.tornado.httpclient.AsyncHTTPClient.configure(None)
+            tornado.httpclient.AsyncHTTPClient.configure(None)
             client_argspec = salt.utils.args.get_function_argspec(
-                salt.ext.tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize
+                tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize
             )
 
         supports_max_body_size = "max_body_size" in client_argspec.args
@@ -615,7 +616,7 @@ def query(
                 else HTTPClient()
             )
             result = download_client.fetch(url_full, **req_kwargs)
-        except salt.ext.tornado.httpclient.HTTPError as exc:
+        except tornado.httpclient.HTTPError as exc:
             ret["status"] = exc.code
             ret["error"] = str(exc)
             return ret
diff --git a/salt/utils/process.py b/salt/utils/process.py
index 6c393706acd..e1ee5ff83df 100644
--- a/salt/utils/process.py
+++ b/salt/utils/process.py
@@ -20,13 +20,14 @@ import sys
 import threading
 import time
 
+from tornado import gen
+
 import salt._logging
 import salt.defaults.exitcodes
 import salt.utils.files
 import salt.utils.path
 import salt.utils.platform
 import salt.utils.versions
-from salt.ext.tornado import gen
 
 log = logging.getLogger(__name__)
 
diff --git a/salt/utils/thin.py b/salt/utils/thin.py
index 80766a286a9..e6f75041db5 100644
--- a/salt/utils/thin.py
+++ b/salt/utils/thin.py
@@ -20,11 +20,11 @@ import jinja2
 import looseversion
 import msgpack
 import packaging
+import tornado
 import yaml
 
 import salt
 import salt.exceptions
-import salt.ext.tornado as tornado
 import salt.utils.files
 import salt.utils.hashutils
 import salt.utils.json
diff --git a/tests/integration/minion/test_minion_cache.py b/tests/integration/minion/test_minion_cache.py
index 039c1e51c94..33125f8bccb 100644
--- a/tests/integration/minion/test_minion_cache.py
+++ b/tests/integration/minion/test_minion_cache.py
@@ -35,7 +35,7 @@ class BasePillarTest(ModuleCase):
         }
         with patch("salt.loader.grains", return_value={}), patch(
             "salt.minion.SMinion.gen_modules"
-        ), patch("salt.ext.tornado.ioloop.IOLoop.current"):
+        ), patch("tornado.ioloop.IOLoop.current"):
             minion = salt.minion.SMinion(opts)
             self.assertTrue("pillar" in os.listdir(tempdir))
             pillar_cache = os.path.join(tempdir, "pillar")
diff --git a/tests/integration/modules/test_gem.py b/tests/integration/modules/test_gem.py
index f484e8c3c1d..fca9fbcf7ca 100644
--- a/tests/integration/modules/test_gem.py
+++ b/tests/integration/modules/test_gem.py
@@ -3,8 +3,8 @@ Integration tests for Ruby Gem module
 """
 
 import pytest
+from tornado.httpclient import HTTPClient
 
-from salt.ext.tornado.httpclient import HTTPClient
 from tests.support.case import ModuleCase
 
 
diff --git a/tests/integration/netapi/rest_tornado/test_app.py b/tests/integration/netapi/rest_tornado/test_app.py
index 65bb4ad6463..2b020da4ac2 100644
--- a/tests/integration/netapi/rest_tornado/test_app.py
+++ b/tests/integration/netapi/rest_tornado/test_app.py
@@ -3,13 +3,13 @@ import threading
 import time
 
 import pytest
+import tornado.escape
+import tornado.web
+from tornado.testing import AsyncHTTPTestCase
 
 import salt.auth
-import salt.ext.tornado.escape
-import salt.ext.tornado.web
 import salt.utils.json
 import salt.utils.stringutils
-from salt.ext.tornado.testing import AsyncHTTPTestCase
 from salt.netapi.rest_tornado import saltnado
 from tests.support.helpers import TstSuiteLoggingHandler, patched_environ
 from tests.support.mixins import AdaptedConfigurationTestCaseMixin
@@ -96,7 +96,7 @@ class SaltnadoIntegrationTestsBase(
             del self.patched_environ
 
     def build_tornado_app(self, urls):
-        application = salt.ext.tornado.web.Application(urls, debug=True)
+        application = tornado.web.Application(urls, debug=True)
 
         application.auth = self.auth
         application.opts = self.opts
@@ -112,11 +112,11 @@ class SaltnadoIntegrationTestsBase(
             if response.headers.get("Content-Type") == "application/json":
                 response._body = response.body.decode("utf-8")
             else:
-                response._body = salt.ext.tornado.escape.native_str(response.body)
+                response._body = tornado.escape.native_str(response.body)
         return response
 
-    def fetch(self, path, **kwargs):
-        return self.decode_body(super().fetch(path, **kwargs))
+    def fetch(self, path, raise_error=False, **kwargs):
+        return self.decode_body(super().fetch(path, raise_error=raise_error, **kwargs))
 
     def get_app(self):
         raise NotImplementedError
diff --git a/tests/pytests/conftest.py b/tests/pytests/conftest.py
index a864c0972e2..e249e66bfa0 100644
--- a/tests/pytests/conftest.py
+++ b/tests/pytests/conftest.py
@@ -17,10 +17,10 @@ import types
 
 import attr
 import pytest
+import tornado.ioloop
 from pytestshellutils.utils import ports
 from saltfactories.utils import random_string
 
-import salt.ext.tornado.ioloop
 import salt.utils.files
 import salt.utils.platform
 from salt.serializers import yaml
@@ -603,12 +603,12 @@ def pytest_pyfunc_call(pyfuncitem):
     try:
         loop = funcargs["io_loop"]
     except KeyError:
-        loop = salt.ext.tornado.ioloop.IOLoop.current()
+        loop = tornado.ioloop.IOLoop.current()
         if loop.closed():
             log.warning("IOLoop found to be closed when starting test")
             loop = asyncio.new_event_loop()
             asyncio.set_event_loop(loop)
-            loop = salt.ext.tornado.ioloop.IOLoop.current()
+            loop = tornado.ioloop.IOLoop.current()
 
     __tracebackhide__ = True
 
@@ -625,7 +625,7 @@ def io_loop():
     """
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
-    loop = salt.ext.tornado.ioloop.IOLoop.current()
+    loop = tornado.ioloop.IOLoop.current()
     loop.make_current()
     try:
         yield loop
diff --git a/tests/pytests/functional/channel/test_server.py b/tests/pytests/functional/channel/test_server.py
index 86415cafa27..7cbfff78d87 100644
--- a/tests/pytests/functional/channel/test_server.py
+++ b/tests/pytests/functional/channel/test_server.py
@@ -9,14 +9,14 @@ import time
 from pathlib import Path
 
 import pytest
+import tornado.gen
+import tornado.ioloop
 from pytestshellutils.utils import ports
 from saltfactories.utils import random_string
 
 import salt.channel.client
 import salt.channel.server
 import salt.config
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
 import salt.master
 import salt.utils.platform
 import salt.utils.process
@@ -124,7 +124,7 @@ def master_secrets():
     salt.master.SMaster.secrets.pop("aes")
 
 
-@salt.ext.tornado.gen.coroutine
+@tornado.gen.coroutine
 def _connect_and_publish(
     io_loop, channel_minion_id, channel, server, received, timeout=60
 ):
@@ -141,7 +141,7 @@ def _connect_and_publish(
     server.publish({"tgt_type": "glob", "tgt": [channel_minion_id], "WTF": "SON"})
     start = time.time()
     while time.time() - start < timeout:
-        yield salt.ext.tornado.gen.sleep(1)
+        yield tornado.gen.sleep(1)
     io_loop.stop()
 
 
diff --git a/tests/pytests/functional/netapi/rest_cherrypy/conftest.py b/tests/pytests/functional/netapi/rest_cherrypy/conftest.py
index e25ce0cf13f..ef467a58945 100644
--- a/tests/pytests/functional/netapi/rest_cherrypy/conftest.py
+++ b/tests/pytests/functional/netapi/rest_cherrypy/conftest.py
@@ -1,6 +1,6 @@
 import pytest
+import tornado.wsgi
 
-import salt.ext.tornado.wsgi
 import salt.netapi.rest_cherrypy.app
 import tests.support.netapi as netapi
 from tests.support.mock import patch
@@ -23,7 +23,7 @@ def app(client_config, load_auth):
     with patch("salt.netapi.NetapiClient._is_master_running", return_value=True), patch(
         "salt.auth.Resolver.mk_token", load_auth.mk_token
     ):
-        yield salt.ext.tornado.wsgi.WSGIContainer(
+        yield tornado.wsgi.WSGIContainer(
             cherrypy.Application(app, "/", config=cherry_opts)
         )
 
diff --git a/tests/pytests/functional/netapi/rest_cherrypy/test_auth.py b/tests/pytests/functional/netapi/rest_cherrypy/test_auth.py
index eb6e155487b..bd348430bf2 100644
--- a/tests/pytests/functional/netapi/rest_cherrypy/test_auth.py
+++ b/tests/pytests/functional/netapi/rest_cherrypy/test_auth.py
@@ -1,9 +1,9 @@
 import urllib.parse
 
 import pytest
+from tornado.httpclient import HTTPError
 
 import salt.utils.json
-from salt.ext.tornado.httpclient import HTTPError
 
 
 async def test_get_root_noauth(http_client):
diff --git a/tests/pytests/functional/netapi/rest_cherrypy/test_auth_pam.py b/tests/pytests/functional/netapi/rest_cherrypy/test_auth_pam.py
index f275b1510e3..bcb1f764b10 100644
--- a/tests/pytests/functional/netapi/rest_cherrypy/test_auth_pam.py
+++ b/tests/pytests/functional/netapi/rest_cherrypy/test_auth_pam.py
@@ -1,8 +1,7 @@
 import urllib.parse
 
 import pytest
-
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
 
 pytestmark = [
     pytest.mark.destructive_test,
diff --git a/tests/pytests/functional/netapi/rest_cherrypy/test_out_formats.py b/tests/pytests/functional/netapi/rest_cherrypy/test_out_formats.py
index 444bced9be1..d70c4e9d60b 100644
--- a/tests/pytests/functional/netapi/rest_cherrypy/test_out_formats.py
+++ b/tests/pytests/functional/netapi/rest_cherrypy/test_out_formats.py
@@ -1,6 +1,5 @@
 import pytest
-
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
 
 
 @pytest.fixture
diff --git a/tests/pytests/functional/netapi/rest_tornado/test_auth_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_auth_handler.py
index 91e04ee0578..d3f42af9621 100644
--- a/tests/pytests/functional/netapi/rest_tornado/test_auth_handler.py
+++ b/tests/pytests/functional/netapi/rest_tornado/test_auth_handler.py
@@ -1,10 +1,10 @@
 import urllib.parse
 
 import pytest
+from tornado.httpclient import HTTPError
 
 import salt.utils.json
 import salt.utils.yaml
-from salt.ext.tornado.httpclient import HTTPError
 from salt.netapi.rest_tornado import saltnado
 
 pytestmark = [
diff --git a/tests/pytests/functional/netapi/rest_tornado/test_auth_handler_pam.py b/tests/pytests/functional/netapi/rest_tornado/test_auth_handler_pam.py
index ffd9871339b..b2c514e9586 100644
--- a/tests/pytests/functional/netapi/rest_tornado/test_auth_handler_pam.py
+++ b/tests/pytests/functional/netapi/rest_tornado/test_auth_handler_pam.py
@@ -1,10 +1,10 @@
 import urllib.parse
 
 import pytest
+from tornado.httpclient import HTTPError
 
 import salt.utils.json
 import salt.utils.yaml
-from salt.ext.tornado.httpclient import HTTPError
 from salt.netapi.rest_tornado import saltnado
 
 pytestmark = [
diff --git a/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py
index 837bde8bfa1..a8c4b762f6e 100644
--- a/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py
+++ b/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py
@@ -1,10 +1,10 @@
 import urllib.parse
 
 import pytest
+from tornado.httpclient import HTTPError
 
 import salt.utils.json
 import salt.utils.yaml
-from salt.ext.tornado.httpclient import HTTPError
 from salt.netapi.rest_tornado import saltnado
 
 
diff --git a/tests/pytests/functional/netapi/rest_tornado/test_utils.py b/tests/pytests/functional/netapi/rest_tornado/test_utils.py
index b9c72912371..49b50a0c36e 100644
--- a/tests/pytests/functional/netapi/rest_tornado/test_utils.py
+++ b/tests/pytests/functional/netapi/rest_tornado/test_utils.py
@@ -1,4 +1,5 @@
-import salt.ext.tornado.concurrent
+import tornado.concurrent
+
 from salt.netapi.rest_tornado import saltnado
 
 
@@ -9,7 +10,7 @@ async def test_any_future():
     # create a few futures
     futures = []
     for _ in range(3):
-        future = salt.ext.tornado.concurrent.Future()
+        future = tornado.concurrent.Future()
         futures.append(future)
 
     # create an any future, make sure it isn't immediately done
diff --git a/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py
index 3f40c765489..34f33e5f817 100644
--- a/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py
+++ b/tests/pytests/functional/netapi/rest_tornado/test_webhooks_handler.py
@@ -1,8 +1,8 @@
 import urllib.parse
 
 import pytest
+import tornado
 
-import salt.ext.tornado
 import salt.utils.json
 from salt.netapi.rest_tornado import saltnado
 from tests.support.mock import MagicMock, patch
@@ -38,7 +38,7 @@ async def test_hook_can_handle_get_parameters(http_client, app, content_type_map
                         "Content-Type": "application/json",
                         "Host": host,
                         "Accept-Encoding": "gzip",
-                        "User-Agent": f"Tornado/{salt.ext.tornado.version}",
+                        "User-Agent": f"Tornado/{tornado.version}",
                     },
                     "post": {},
                     "get": {"param": ["1", "2"]},
diff --git a/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py
index 657f5770a3b..b7cf3a4a37c 100644
--- a/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py
+++ b/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py
@@ -2,12 +2,12 @@ import hashlib
 import urllib.parse
 
 import pytest
+from tornado.httpclient import HTTPError, HTTPRequest
+from tornado.websocket import websocket_connect
 
 import salt.netapi.rest_tornado as rest_tornado
 import salt.utils.json
 import salt.utils.yaml
-from salt.ext.tornado.httpclient import HTTPError, HTTPRequest
-from salt.ext.tornado.websocket import websocket_connect
 
 pytestmark = [
     pytest.mark.destructive_test,
diff --git a/tests/pytests/functional/transport/ipc/test_client.py b/tests/pytests/functional/transport/ipc/test_client.py
index df6f2372011..20bc43aa064 100644
--- a/tests/pytests/functional/transport/ipc/test_client.py
+++ b/tests/pytests/functional/transport/ipc/test_client.py
@@ -2,10 +2,10 @@ import pathlib
 
 import attr
 import pytest
+from tornado import locks
 
 import salt.transport.ipc
 import salt.utils.platform
-from salt.ext.tornado import locks
 
 pytestmark = [
     # Windows does not support POSIX IPC
diff --git a/tests/pytests/functional/transport/ipc/test_subscriber.py b/tests/pytests/functional/transport/ipc/test_subscriber.py
index 07676305cb4..cd7865ada9f 100644
--- a/tests/pytests/functional/transport/ipc/test_subscriber.py
+++ b/tests/pytests/functional/transport/ipc/test_subscriber.py
@@ -2,12 +2,12 @@ import pathlib
 
 import attr
 import pytest
+import tornado.gen
+from tornado import locks
 
 import salt.channel.server
-import salt.ext.tornado.gen
 import salt.transport.ipc
 import salt.utils.platform
-from salt.ext.tornado import locks
 
 pytestmark = [
     # Windows does not support POSIX IPC
@@ -110,9 +110,9 @@ async def test_basic_send(channel):
     # XXX: IPCClient connect and connected methods need to be cleaned up as
     # this should not be needed.
     while not channel.subscriber._connecting_future.done():
-        await salt.ext.tornado.gen.sleep(0.01)
+        await tornado.gen.sleep(0.01)
     while not channel.subscriber.connected():
-        await salt.ext.tornado.gen.sleep(0.01)
+        await tornado.gen.sleep(0.01)
     assert channel.subscriber.connected()
     await channel.publish(msg)
     ret = await channel.read()
diff --git a/tests/pytests/functional/transport/server/test_req_channel.py b/tests/pytests/functional/transport/server/test_req_channel.py
index 46a3b2fe0e5..1ed69355baf 100644
--- a/tests/pytests/functional/transport/server/test_req_channel.py
+++ b/tests/pytests/functional/transport/server/test_req_channel.py
@@ -3,13 +3,13 @@ import logging
 import multiprocessing
 
 import pytest
+import tornado.gen
 from pytestshellutils.utils.processes import terminate_process
 
 import salt.channel.client
 import salt.channel.server
 import salt.config
 import salt.exceptions
-import salt.ext.tornado.gen
 import salt.master
 import salt.utils.platform
 import salt.utils.process
@@ -55,7 +55,7 @@ class ReqServerChannelProcess(salt.utils.process.SignalHandlingProcess):
             ),
         }
 
-        self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+        self.io_loop = tornado.ioloop.IOLoop()
         self.io_loop.make_current()
         self.req_server_channel.post_fork(self._handle_payload, io_loop=self.io_loop)
         self.io_loop.add_callback(self.running.set)
@@ -91,11 +91,11 @@ class ReqServerChannelProcess(salt.utils.process.SignalHandlingProcess):
                 terminate_process(pid=pid, kill_children=True, slow_stop=False)
             self.process_manager = None
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _handle_payload(self, payload):
         if self.req_channel_crypt == "clear":
-            raise salt.ext.tornado.gen.Return((payload, {"fun": "send_clear"}))
-        raise salt.ext.tornado.gen.Return((payload, {"fun": "send"}))
+            raise tornado.gen.Return((payload, {"fun": "send_clear"}))
+        raise tornado.gen.Return((payload, {"fun": "send"}))
 
 
 @pytest.fixture
diff --git a/tests/pytests/functional/transport/tcp/test_message_client.py b/tests/pytests/functional/transport/tcp/test_message_client.py
index 292f389000d..09864f7cfa2 100644
--- a/tests/pytests/functional/transport/tcp/test_message_client.py
+++ b/tests/pytests/functional/transport/tcp/test_message_client.py
@@ -1,10 +1,10 @@
 import logging
 
 import pytest
+import tornado.gen
+import tornado.iostream
+import tornado.tcpserver
 
-import salt.ext.tornado.gen
-import salt.ext.tornado.iostream
-import salt.ext.tornado.tcpserver
 import salt.transport.tcp
 import salt.utils.msgpack
 
@@ -21,7 +21,7 @@ def config():
 
 @pytest.fixture
 def server(config):
-    class TestServer(salt.ext.tornado.tcpserver.TCPServer):
+    class TestServer(tornado.tcpserver.TCPServer):
         send = []
         disconnect = False
 
@@ -31,10 +31,10 @@ def server(config):
                     msg = self.send.pop(0)
                     try:
                         await stream.write(msg)
-                    except salt.ext.tornado.iostream.StreamClosedError:
+                    except tornado.iostream.StreamClosedError:
                         break
                 else:
-                    await salt.ext.tornado.gen.sleep(1)
+                    await tornado.gen.sleep(1)
             stream.close()
 
     server = TestServer()
@@ -81,14 +81,14 @@ async def test_message_client_reconnect(io_loop, config, client, server):
     server.send.append(partial)
 
     while not received:
-        await salt.ext.tornado.gen.sleep(1)
+        await tornado.gen.sleep(1)
     assert received == [msg]
 
     # The message client has unpacked one msg and there is a partial msg left in
     # the unpacker. Closing the stream now leaves the unpacker in a bad state
     # since the rest of the partil message will never be received.
     server.disconnect = True
-    await salt.ext.tornado.gen.sleep(1)
+    await tornado.gen.sleep(1)
     server.disconnect = False
     received = []
 
@@ -97,5 +97,5 @@ async def test_message_client_reconnect(io_loop, config, client, server):
     # rest of this test would fail.
     server.send.append(pmsg)
     while not received:
-        await salt.ext.tornado.gen.sleep(1)
+        await tornado.gen.sleep(1)
     assert received == [msg, msg]
diff --git a/tests/pytests/functional/utils/test_async_event_publisher.py b/tests/pytests/functional/utils/test_async_event_publisher.py
index aa773b452ab..ea9bcacb3b7 100644
--- a/tests/pytests/functional/utils/test_async_event_publisher.py
+++ b/tests/pytests/functional/utils/test_async_event_publisher.py
@@ -2,7 +2,6 @@ import pytest
 import zmq
 
 import salt.config
-import salt.ext.tornado.ioloop
 import salt.utils.event
 import salt.utils.stringutils
 
diff --git a/tests/pytests/integration/netapi/rest_cherrypy/conftest.py b/tests/pytests/integration/netapi/rest_cherrypy/conftest.py
index bc0eb017bce..071cea3a2d7 100644
--- a/tests/pytests/integration/netapi/rest_cherrypy/conftest.py
+++ b/tests/pytests/integration/netapi/rest_cherrypy/conftest.py
@@ -1,6 +1,6 @@
 import pytest
+import tornado.wsgi
 
-import salt.ext.tornado.wsgi
 import salt.netapi.rest_cherrypy.app
 import tests.support.netapi as netapi
 
@@ -22,7 +22,7 @@ def client_config(client_config, netapi_port, request):
 def app(client_config, load_auth, salt_minion):
     app, _, cherry_opts = salt.netapi.rest_cherrypy.app.get_app(client_config)
 
-    return salt.ext.tornado.wsgi.WSGIContainer(
+    return tornado.wsgi.WSGIContainer(
         cherrypy.Application(app, "/", config=cherry_opts)
     )
 
diff --git a/tests/pytests/integration/netapi/rest_cherrypy/test_auth.py b/tests/pytests/integration/netapi/rest_cherrypy/test_auth.py
index ab58d70b5ed..ce11207d2c1 100644
--- a/tests/pytests/integration/netapi/rest_cherrypy/test_auth.py
+++ b/tests/pytests/integration/netapi/rest_cherrypy/test_auth.py
@@ -1,9 +1,9 @@
 import urllib.parse
 
 import pytest
+from tornado.httpclient import HTTPError
 
 import salt.utils.json
-from salt.ext.tornado.httpclient import HTTPError
 
 
 async def test_get_root_noauth(http_client):
diff --git a/tests/pytests/integration/netapi/rest_cherrypy/test_run.py b/tests/pytests/integration/netapi/rest_cherrypy/test_run.py
index e012d13c593..61644331b31 100644
--- a/tests/pytests/integration/netapi/rest_cherrypy/test_run.py
+++ b/tests/pytests/integration/netapi/rest_cherrypy/test_run.py
@@ -1,8 +1,7 @@
 import urllib.parse
 
 import pytest
-
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
 
 
 @pytest.mark.netapi_client_data(["local"])
diff --git a/tests/pytests/integration/netapi/rest_tornado/test_events_api_handler.py b/tests/pytests/integration/netapi/rest_tornado/test_events_api_handler.py
index 188e4603492..bc898f1ce12 100644
--- a/tests/pytests/integration/netapi/rest_tornado/test_events_api_handler.py
+++ b/tests/pytests/integration/netapi/rest_tornado/test_events_api_handler.py
@@ -1,8 +1,8 @@
 from functools import partial
 
 import pytest
+import tornado.gen
 
-import salt.ext.tornado.gen
 from salt.netapi.rest_tornado import saltnado
 
 # TODO: run all the same tests from the root handler, but for now since they are
@@ -45,6 +45,6 @@ async def test_get(http_client, io_loop, app):
     )
 
     while len(events_fired) < 5:
-        await salt.ext.tornado.gen.sleep(1)
+        await tornado.gen.sleep(1)
 
     assert len(events_fired) >= 5
diff --git a/tests/pytests/integration/netapi/rest_tornado/test_minions_api_handler.py b/tests/pytests/integration/netapi/rest_tornado/test_minions_api_handler.py
index 05146b45a45..080ba4698da 100644
--- a/tests/pytests/integration/netapi/rest_tornado/test_minions_api_handler.py
+++ b/tests/pytests/integration/netapi/rest_tornado/test_minions_api_handler.py
@@ -1,7 +1,7 @@
 import pytest
+from tornado.httpclient import HTTPError
 
 import salt.utils.json
-from salt.ext.tornado.httpclient import HTTPError
 from salt.netapi.rest_tornado import saltnado
 
 
diff --git a/tests/pytests/integration/netapi/rest_tornado/test_root_handler.py b/tests/pytests/integration/netapi/rest_tornado/test_root_handler.py
index 9dd4cbd6aab..607f60ccfed 100644
--- a/tests/pytests/integration/netapi/rest_tornado/test_root_handler.py
+++ b/tests/pytests/integration/netapi/rest_tornado/test_root_handler.py
@@ -1,7 +1,7 @@
 import pytest
+from tornado.httpclient import HTTPError
 
 import salt.utils.json
-from salt.ext.tornado.httpclient import HTTPError
 from salt.netapi.rest_tornado import saltnado
 
 
diff --git a/tests/pytests/unit/fileserver/gitfs/test_gitfs.py b/tests/pytests/unit/fileserver/gitfs/test_gitfs.py
index 4c7e8dd7c5c..55745cf32a3 100644
--- a/tests/pytests/unit/fileserver/gitfs/test_gitfs.py
+++ b/tests/pytests/unit/fileserver/gitfs/test_gitfs.py
@@ -24,8 +24,8 @@ import os
 import pathlib
 
 import pytest
+import tornado.ioloop
 
-import salt.ext.tornado.ioloop
 import salt.fileserver.gitfs as gitfs
 import salt.utils.files
 import salt.utils.gitfs
@@ -126,9 +126,7 @@ def testfile(tmp_path):
 @pytest.fixture
 def repo_dir(tmp_path, unicode_dirname, tag_name, unicode_filename):
     try:
-        del salt.utils.gitfs.GitFS.instance_map[
-            salt.ext.tornado.ioloop.IOLoop.current()
-        ]
+        del salt.utils.gitfs.GitFS.instance_map[tornado.ioloop.IOLoop.current()]
     except KeyError:
         pass
 
diff --git a/tests/pytests/unit/fileserver/gitfs/test_gitfs_config.py b/tests/pytests/unit/fileserver/gitfs/test_gitfs_config.py
index afd0cfdc9e2..209e659c6dd 100644
--- a/tests/pytests/unit/fileserver/gitfs/test_gitfs_config.py
+++ b/tests/pytests/unit/fileserver/gitfs/test_gitfs_config.py
@@ -1,8 +1,8 @@
 import textwrap
 
 import pytest
+import tornado.ioloop
 
-import salt.ext.tornado.ioloop
 import salt.fileserver.gitfs as gitfs
 import salt.utils.files
 import salt.utils.gitfs
@@ -73,9 +73,7 @@ def configure_loader_modules(tmp_path):
 @pytest.fixture(scope="module", autouse=True)
 def clear_instance_map():
     try:
-        del salt.utils.gitfs.GitFS.instance_map[
-            salt.ext.tornado.ioloop.IOLoop.current()
-        ]
+        del salt.utils.gitfs.GitFS.instance_map[tornado.ioloop.IOLoop.current()]
     except KeyError:
         pass
 
diff --git a/tests/pytests/unit/test_ext_importers.py b/tests/pytests/unit/test_ext_importers.py
deleted file mode 100644
index 02ee700bdaf..00000000000
--- a/tests/pytests/unit/test_ext_importers.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import logging
-import os
-import subprocess
-import sys
-
-import pytest
-
-import salt
-
-log = logging.getLogger(__name__)
-
-
-def test_tornado_import_override(tmp_path):
-    """
-    Ensure we are not using any non vendor'ed tornado
-    """
-    test_source = """
-    from __future__ import absolute_import, print_function
-    import salt
-    import tornado
-    print(tornado.__name__)
-    """
-    tornado_source = """
-    foo = 'bar'
-    """
-    with pytest.helpers.temp_file(
-        "test.py", directory=tmp_path, contents=test_source
-    ) as test_source_path, pytest.helpers.temp_file(
-        "tornado.py", directory=tmp_path, contents=tornado_source
-    ):
-        env = os.environ.copy()
-        env["PYTHONPATH"] = os.pathsep.join(sys.path)
-        ret = subprocess.run(
-            [sys.executable, str(test_source_path)],
-            stderr=subprocess.PIPE,
-            stdout=subprocess.PIPE,
-            env=env,
-            shell=False,
-            check=False,
-            universal_newlines=True,
-        )
-        assert ret.returncode == 0
-        if salt.USE_VENDORED_TORNADO:
-            assert ret.stdout.strip() == "salt.ext.tornado"
-        else:
-            assert ret.stdout.strip() == "tornado"
-
-
-def test_regression_56063():
-    importer = salt.TornadoImporter()
-    try:
-        importer.find_module("tornado")
-    except TypeError:
-        assert False, "TornadoImporter raised type error when one argument passed"
diff --git a/tests/pytests/unit/test_minion.py b/tests/pytests/unit/test_minion.py
index 1cee025a485..3f2b98c2245 100644
--- a/tests/pytests/unit/test_minion.py
+++ b/tests/pytests/unit/test_minion.py
@@ -3,10 +3,10 @@ import logging
 import os
 
 import pytest
+import tornado
+import tornado.gen
+import tornado.testing
 
-import salt.ext.tornado
-import salt.ext.tornado.gen
-import salt.ext.tornado.testing
 import salt.minion
 import salt.syspaths
 import salt.utils.crypt
@@ -63,9 +63,7 @@ def test_minion_load_grains_default():
         ),
         (
             "fire_event_async",
-            lambda data, tag, cb=None, timeout=60: salt.ext.tornado.gen.maybe_future(
-                True
-            ),
+            lambda data, tag, cb=None, timeout=60: tornado.gen.maybe_future(True),
         ),
     ],
 )
@@ -121,7 +119,7 @@ def test_send_req_fires_completion_event(event, minion_opts):
 async def test_send_req_async_regression_62453(minion_opts):
     event_enter = MagicMock()
     event_enter.send.side_effect = (
-        lambda data, tag, cb=None, timeout=60: salt.ext.tornado.gen.maybe_future(True)
+        lambda data, tag, cb=None, timeout=60: tornado.gen.maybe_future(True)
     )
     event = MagicMock()
     event.__enter__.return_value = event_enter
@@ -317,7 +315,7 @@ def test_handle_decoded_payload_jid_match_in_jid_queue(minion_opts):
     minion = salt.minion.Minion(
         minion_opts,
         jid_queue=copy.copy(mock_jid_queue),
-        io_loop=salt.ext.tornado.ioloop.IOLoop(),
+        io_loop=tornado.ioloop.IOLoop(),
     )
     try:
         ret = minion._handle_decoded_payload(mock_data).result()
@@ -346,7 +344,7 @@ def test_handle_decoded_payload_jid_queue_addition(minion_opts):
         minion = salt.minion.Minion(
             minion_opts,
             jid_queue=copy.copy(mock_jid_queue),
-            io_loop=salt.ext.tornado.ioloop.IOLoop(),
+            io_loop=tornado.ioloop.IOLoop(),
         )
         try:
 
@@ -383,7 +381,7 @@ def test_handle_decoded_payload_jid_queue_reduced_minion_jid_queue_hwm(minion_op
         minion = salt.minion.Minion(
             minion_opts,
             jid_queue=copy.copy(mock_jid_queue),
-            io_loop=salt.ext.tornado.ioloop.IOLoop(),
+            io_loop=tornado.ioloop.IOLoop(),
         )
         try:
 
@@ -415,15 +413,15 @@ def test_process_count_max(minion_opts):
     ), patch(
         "salt.utils.minion.running", MagicMock(return_value=[])
     ), patch(
-        "salt.ext.tornado.gen.sleep",
-        MagicMock(return_value=salt.ext.tornado.concurrent.Future()),
+        "tornado.gen.sleep",
+        MagicMock(return_value=tornado.concurrent.Future()),
     ):
         process_count_max = 10
         minion_opts["__role"] = "minion"
         minion_opts["minion_jid_queue_hwm"] = 100
         minion_opts["process_count_max"] = process_count_max
 
-        io_loop = salt.ext.tornado.ioloop.IOLoop()
+        io_loop = tornado.ioloop.IOLoop()
         minion = salt.minion.Minion(minion_opts, jid_queue=[], io_loop=io_loop)
         try:
 
@@ -431,9 +429,7 @@ def test_process_count_max(minion_opts):
             class SleepCalledException(Exception):
                 """Thrown when sleep is called"""
 
-            salt.ext.tornado.gen.sleep.return_value.set_exception(
-                SleepCalledException()
-            )
+            tornado.gen.sleep.return_value.set_exception(SleepCalledException())
 
             # up until process_count_max: gen.sleep does not get called, processes are started normally
             for i in range(process_count_max):
@@ -481,7 +477,7 @@ def test_beacons_before_connect(minion_opts):
         MagicMock(return_value=True),
     ):
         minion_opts["beacons_before_connect"] = True
-        io_loop = salt.ext.tornado.ioloop.IOLoop()
+        io_loop = tornado.ioloop.IOLoop()
         io_loop.make_current()
         minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
         try:
@@ -514,7 +510,7 @@ def test_scheduler_before_connect(minion_opts):
         MagicMock(return_value=True),
     ):
         minion_opts["scheduler_before_connect"] = True
-        io_loop = salt.ext.tornado.ioloop.IOLoop()
+        io_loop = tornado.ioloop.IOLoop()
         io_loop.make_current()
         minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
         try:
@@ -545,7 +541,7 @@ def test_minion_module_refresh(minion_opts):
         try:
             minion = salt.minion.Minion(
                 minion_opts,
-                io_loop=salt.ext.tornado.ioloop.IOLoop(),
+                io_loop=tornado.ioloop.IOLoop(),
             )
             minion.schedule = salt.utils.schedule.Schedule(
                 minion_opts, {}, returners={}
@@ -573,7 +569,7 @@ def test_minion_module_refresh_beacons_refresh(minion_opts):
         try:
             minion = salt.minion.Minion(
                 minion_opts,
-                io_loop=salt.ext.tornado.ioloop.IOLoop(),
+                io_loop=tornado.ioloop.IOLoop(),
             )
             minion.schedule = salt.utils.schedule.Schedule(
                 minion_opts, {}, returners={}
@@ -603,7 +599,7 @@ def test_when_ping_interval_is_set_the_callback_should_be_added_to_periodic_call
         MagicMock(return_value=True),
     ):
         minion_opts["ping_interval"] = 10
-        io_loop = salt.ext.tornado.ioloop.IOLoop()
+        io_loop = tornado.ioloop.IOLoop()
         io_loop.make_current()
         minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
         try:
@@ -625,7 +621,7 @@ def test_when_passed_start_event_grains(minion_opts):
     # provide mock opts an os grain since we'll look for it later.
     minion_opts["grains"]["os"] = "linux"
     minion_opts["start_event_grains"] = ["os"]
-    io_loop = salt.ext.tornado.ioloop.IOLoop()
+    io_loop = tornado.ioloop.IOLoop()
     io_loop.make_current()
     minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
     try:
@@ -644,7 +640,7 @@ def test_when_passed_start_event_grains(minion_opts):
 
 @pytest.mark.slow_test
 def test_when_not_passed_start_event_grains(minion_opts):
-    io_loop = salt.ext.tornado.ioloop.IOLoop()
+    io_loop = tornado.ioloop.IOLoop()
     io_loop.make_current()
     minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
     try:
@@ -661,7 +657,7 @@ def test_when_not_passed_start_event_grains(minion_opts):
 @pytest.mark.slow_test
 def test_when_other_events_fired_and_start_event_grains_are_set(minion_opts):
     minion_opts["start_event_grains"] = ["os"]
-    io_loop = salt.ext.tornado.ioloop.IOLoop()
+    io_loop = tornado.ioloop.IOLoop()
     io_loop.make_current()
     minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
     try:
@@ -699,7 +695,7 @@ def test_gen_modules_executors(minion_opts):
     """
     Ensure gen_modules is called with the correct arguments #54429
     """
-    io_loop = salt.ext.tornado.ioloop.IOLoop()
+    io_loop = tornado.ioloop.IOLoop()
     io_loop.make_current()
     minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
 
@@ -723,7 +719,7 @@ def test_reinit_crypto_on_fork(minion_opts):
     minion_opts["multiprocessing"] = True
     with patch("salt.utils.process.default_signals"):
 
-        io_loop = salt.ext.tornado.ioloop.IOLoop()
+        io_loop = tornado.ioloop.IOLoop()
         io_loop.make_current()
         minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
 
@@ -764,7 +760,7 @@ def test_minion_manage_schedule(minion_opts):
         "salt.utils.process.SignalHandlingProcess.join",
         MagicMock(return_value=True),
     ):
-        io_loop = salt.ext.tornado.ioloop.IOLoop()
+        io_loop = tornado.ioloop.IOLoop()
         io_loop.make_current()
 
         with patch("salt.utils.schedule.clean_proc_dir", MagicMock(return_value=None)):
@@ -824,7 +820,7 @@ def test_minion_manage_beacons(minion_opts):
         try:
             minion_opts["beacons"] = {}
 
-            io_loop = salt.ext.tornado.ioloop.IOLoop()
+            io_loop = tornado.ioloop.IOLoop()
             io_loop.make_current()
 
             mock_functions = {"test.ping": None}
@@ -992,7 +988,7 @@ def test_minion_grains_refresh_pre_exec_false(minion_opts):
         minion = salt.minion.Minion(
             minion_opts,
             jid_queue=None,
-            io_loop=salt.ext.tornado.ioloop.IOLoop(),
+            io_loop=tornado.ioloop.IOLoop(),
             load_grains=False,
         )
         try:
@@ -1015,7 +1011,7 @@ def test_minion_grains_refresh_pre_exec_true(minion_opts):
         minion = salt.minion.Minion(
             minion_opts,
             jid_queue=None,
-            io_loop=salt.ext.tornado.ioloop.IOLoop(),
+            io_loop=tornado.ioloop.IOLoop(),
             load_grains=False,
         )
         try:
diff --git a/tests/pytests/unit/transport/test_ipc.py b/tests/pytests/unit/transport/test_ipc.py
index 5a687836161..77c0e6f2964 100644
--- a/tests/pytests/unit/transport/test_ipc.py
+++ b/tests/pytests/unit/transport/test_ipc.py
@@ -1,7 +1,7 @@
 import pytest
+import tornado.iostream
 from pytestshellutils.utils import ports
 
-import salt.ext.tornado.iostream
 import salt.transport.ipc
 import salt.utils.asynchronous
 import salt.utils.platform
@@ -31,6 +31,6 @@ async def test_ipc_connect_sync_wrapped(io_loop, tmp_path):
         kwargs={"io_loop": io_loop},
         loop_kwarg="io_loop",
     )
-    with pytest.raises(salt.ext.tornado.iostream.StreamClosedError):
+    with pytest.raises(tornado.iostream.StreamClosedError):
         # Don't `await subscriber.connect()`, that's the purpose of the SyncWrapper
         subscriber.connect()
diff --git a/tests/pytests/unit/transport/test_tcp.py b/tests/pytests/unit/transport/test_tcp.py
index bcfb71f5590..594fc1d327a 100644
--- a/tests/pytests/unit/transport/test_tcp.py
+++ b/tests/pytests/unit/transport/test_tcp.py
@@ -4,11 +4,11 @@ import socket
 
 import attr
 import pytest
+import tornado
 from pytestshellutils.utils import ports
 
 import salt.channel.server
 import salt.exceptions
-import salt.ext.tornado
 import salt.transport.tcp
 from tests.support.mock import MagicMock, PropertyMock, patch
 
@@ -31,9 +31,9 @@ def fake_crypto():
 
 @pytest.fixture
 def fake_authd():
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def return_nothing():
-        raise salt.ext.tornado.gen.Return()
+        raise tornado.gen.Return()
 
     with patch(
         "salt.crypt.AsyncAuth.authenticated", new_callable=PropertyMock
@@ -85,7 +85,7 @@ def test_message_client_cleanup_on_close(client_socket, temp_salt_master):
     """
     test message client cleanup on close
     """
-    orig_loop = salt.ext.tornado.ioloop.IOLoop()
+    orig_loop = tornado.ioloop.IOLoop()
     orig_loop.make_current()
 
     opts = dict(temp_salt_master.config.copy(), transport="tcp")
@@ -241,7 +241,7 @@ def test_tcp_pub_server_channel_publish_filtering_str_list(temp_salt_master):
 
 @pytest.fixture(scope="function")
 def salt_message_client():
-    io_loop_mock = MagicMock(spec=salt.ext.tornado.ioloop.IOLoop)
+    io_loop_mock = MagicMock(spec=tornado.ioloop.IOLoop)
     io_loop_mock.call_later.side_effect = lambda *args, **kwargs: (args, kwargs)
 
     client = salt.transport.tcp.MessageClient(
@@ -346,7 +346,7 @@ def test_timeout_message_unknown_future(salt_message_client):
     # if we do have the actual future stored under the id, but it's none
     # we shouldn't fail as well
     message_id = 1
-    future = salt.ext.tornado.concurrent.Future()
+    future = tornado.concurrent.Future()
     future.attempts = 1
     future.tries = 1
     salt_message_client.send_future_map[message_id] = future
@@ -367,16 +367,16 @@ def xtest_client_reconnect_backoff(client_socket):
         client.close()
         assert t == 5
         return
-        # return salt.ext.tornado.gen.sleep()
+        # return tornado.gen.sleep()
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def connect(*args, **kwargs):
         raise Exception("err")
 
     client._tcp_client.connect = connect
 
     try:
-        with patch("salt.ext.tornado.gen.sleep", side_effect=_sleep):
+        with patch("tornado.gen.sleep", side_effect=_sleep):
             client.io_loop.run_sync(client.connect)
     finally:
         client.close()
@@ -456,20 +456,18 @@ def test_presence_events_callback_passed(temp_salt_master, salt_message_client):
 def test_presence_removed_on_stream_closed():
     opts = {"presence_events": True}
 
-    io_loop_mock = MagicMock(spec=salt.ext.tornado.ioloop.IOLoop)
+    io_loop_mock = MagicMock(spec=tornado.ioloop.IOLoop)
 
     with patch("salt.master.AESFuncs.__init__", return_value=None):
         server = salt.transport.tcp.PubServer(opts, io_loop=io_loop_mock)
         server._closing = True
         server.remove_presence_callback = MagicMock()
 
-    client = salt.transport.tcp.Subscriber(
-        salt.ext.tornado.iostream.IOStream, "1.2.3.4"
-    )
+    client = salt.transport.tcp.Subscriber(tornado.iostream.IOStream, "1.2.3.4")
     client._closing = True
     server.clients = {client}
 
-    io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+    io_loop = tornado.ioloop.IOLoop.current()
     package = {
         "topic_lst": [],
         "payload": "test-payload",
@@ -477,8 +475,8 @@ def test_presence_removed_on_stream_closed():
 
     with patch("salt.transport.frame.frame_msg", return_value="framed-payload"):
         with patch(
-            "salt.ext.tornado.iostream.BaseIOStream.write",
-            side_effect=salt.ext.tornado.iostream.StreamClosedError(),
+            "tornado.iostream.BaseIOStream.write",
+            side_effect=tornado.iostream.StreamClosedError(),
         ):
             io_loop.run_sync(functools.partial(server.publish_payload, package, None))
 
diff --git a/tests/pytests/unit/transport/test_zeromq.py b/tests/pytests/unit/transport/test_zeromq.py
index 10bb4917b83..8561b05b3f9 100644
--- a/tests/pytests/unit/transport/test_zeromq.py
+++ b/tests/pytests/unit/transport/test_zeromq.py
@@ -12,14 +12,14 @@ import time
 import uuid
 
 import pytest
+import tornado.gen
+import tornado.ioloop
 
 import salt.channel.client
 import salt.channel.server
 import salt.config
 import salt.crypt
 import salt.exceptions
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
 import salt.transport.zeromq
 import salt.utils.platform
 import salt.utils.process
@@ -343,14 +343,14 @@ def run_loop_in_thread(loop, evt):
     """
     loop.make_current()
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def stopper():
-        yield salt.ext.tornado.gen.sleep(0.1)
+        yield tornado.gen.sleep(0.1)
         while True:
             if not evt.is_set():
                 loop.stop()
                 break
-            yield salt.ext.tornado.gen.sleep(0.3)
+            yield tornado.gen.sleep(0.3)
 
     loop.add_callback(evt.set)
     loop.add_callback(stopper)
@@ -382,7 +382,7 @@ class MockSaltMinionMaster:
         self.server_channel = salt.channel.server.ReqServerChannel.factory(master_opts)
         self.server_channel.pre_fork(self.process_manager)
 
-        self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+        self.io_loop = tornado.ioloop.IOLoop()
         self.evt = threading.Event()
         self.server_channel.post_fork(self._handle_payload, io_loop=self.io_loop)
         self.server_thread = threading.Thread(
@@ -425,13 +425,13 @@ class MockSaltMinionMaster:
 
     # pylint: enable=W1701
     @classmethod
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _handle_payload(cls, payload):
         """
         TODO: something besides echo
         """
         cls.mock._handle_payload_hook()
-        raise salt.ext.tornado.gen.Return((payload, {"fun": "send_clear"}))
+        raise tornado.gen.Return((payload, {"fun": "send_clear"}))
 
 
 @pytest.mark.parametrize("message", ["", [], ()])
@@ -461,7 +461,7 @@ def test_serverside_exception(temp_salt_minion, temp_salt_master):
     """
     with MockSaltMinionMaster(temp_salt_minion, temp_salt_master) as minion_master:
         with patch.object(minion_master.mock, "_handle_payload_hook") as _mock:
-            _mock.side_effect = salt.ext.tornado.gen.Return(({}, {"fun": "madeup-fun"}))
+            _mock.side_effect = tornado.gen.Return(({}, {"fun": "madeup-fun"}))
             ret = minion_master.channel.send({}, timeout=5, tries=1)
             assert ret == "Server-side exception handling payload"
 
@@ -484,7 +484,7 @@ def test_zeromq_async_pub_channel_publish_port(temp_salt_master):
         sign_pub_messages=False,
     )
     opts["master_uri"] = "tcp://{interface}:{publish_port}".format(**opts)
-    ioloop = salt.ext.tornado.ioloop.IOLoop()
+    ioloop = tornado.ioloop.IOLoop()
     transport = salt.transport.zeromq.PublishClient(opts, ioloop)
     with transport:
         patch_socket = MagicMock(return_value=True)
@@ -526,7 +526,7 @@ def test_zeromq_async_pub_channel_filtering_decode_message_no_match(
     )
     opts["master_uri"] = "tcp://{interface}:{publish_port}".format(**opts)
 
-    ioloop = salt.ext.tornado.ioloop.IOLoop()
+    ioloop = tornado.ioloop.IOLoop()
     channel = salt.transport.zeromq.PublishClient(opts, ioloop)
     with channel:
         with patch(
@@ -573,7 +573,7 @@ def test_zeromq_async_pub_channel_filtering_decode_message(
     )
     opts["master_uri"] = "tcp://{interface}:{publish_port}".format(**opts)
 
-    ioloop = salt.ext.tornado.ioloop.IOLoop()
+    ioloop = tornado.ioloop.IOLoop()
     channel = salt.transport.zeromq.PublishClient(opts, ioloop)
     with channel:
         with patch(
@@ -586,7 +586,7 @@ def test_zeromq_async_pub_channel_filtering_decode_message(
 
 
 def test_req_server_chan_encrypt_v2(pki_dir):
-    loop = salt.ext.tornado.ioloop.IOLoop.current()
+    loop = tornado.ioloop.IOLoop.current()
     opts = {
         "worker_threads": 1,
         "master_uri": "tcp://127.0.0.1:4506",
@@ -630,7 +630,7 @@ def test_req_server_chan_encrypt_v2(pki_dir):
 
 
 def test_req_server_chan_encrypt_v1(pki_dir):
-    loop = salt.ext.tornado.ioloop.IOLoop.current()
+    loop = tornado.ioloop.IOLoop.current()
     opts = {
         "worker_threads": 1,
         "master_uri": "tcp://127.0.0.1:4506",
@@ -732,14 +732,14 @@ async def test_req_chan_decode_data_dict_entry_v2(pki_dir):
     client.auth.crypticle.loads = auth.crypticle.loads
     client.transport = MagicMock()
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def mocksend(msg, timeout=60, tries=3):
         client.transport.msg = msg
         load = client.auth.crypticle.loads(msg["load"])
         ret = server._encrypt_private(
             pillar_data, dictkey, target, nonce=load["nonce"], sign_messages=True
         )
-        raise salt.ext.tornado.gen.Return(ret)
+        raise tornado.gen.Return(ret)
 
     client.transport.send = mocksend
 
@@ -802,10 +802,10 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_nonce(pki_dir):
         pillar_data, dictkey, target, nonce=badnonce, sign_messages=True
     )
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def mocksend(msg, timeout=60, tries=3):
         client.transport.msg = msg
-        raise salt.ext.tornado.gen.Return(ret)
+        raise tornado.gen.Return(ret)
 
     client.transport.send = mocksend
 
@@ -865,7 +865,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_signature(pki_dir):
     client.auth.crypticle.loads = auth.crypticle.loads
     client.transport = MagicMock()
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def mocksend(msg, timeout=60, tries=3):
         client.transport.msg = msg
         load = client.auth.crypticle.loads(msg["load"])
@@ -887,7 +887,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_signature(pki_dir):
         data["pillar"] = {"pillar1": "bar"}
         signed_msg["data"] = salt.payload.dumps(data)
         ret[dictkey] = pcrypt.dumps(signed_msg)
-        raise salt.ext.tornado.gen.Return(ret)
+        raise tornado.gen.Return(ret)
 
     client.transport.send = mocksend
 
@@ -947,7 +947,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_key(pki_dir):
     client.auth.crypticle.loads = auth.crypticle.loads
     client.transport = MagicMock()
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def mocksend(msg, timeout=60, tries=3):
         client.transport.msg = msg
         load = client.auth.crypticle.loads(msg["load"])
@@ -976,7 +976,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_key(pki_dir):
         else:
             cipher = PKCS1_OAEP.new(pub)
             ret["key"] = cipher.encrypt(key)
-        raise salt.ext.tornado.gen.Return(ret)
+        raise tornado.gen.Return(ret)
 
     client.transport.send = mocksend
 
diff --git a/tests/pytests/unit/utils/event/test_event.py b/tests/pytests/unit/utils/event/test_event.py
index e289e72dad0..79e73469b00 100644
--- a/tests/pytests/unit/utils/event/test_event.py
+++ b/tests/pytests/unit/utils/event/test_event.py
@@ -5,11 +5,11 @@ import time
 from pathlib import Path
 
 import pytest
+import tornado.ioloop
+import tornado.iostream
 import zmq.eventloop.ioloop
 
 import salt.config
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.iostream
 import salt.utils.event
 import salt.utils.stringutils
 from salt.utils.event import SaltEvent
@@ -302,13 +302,11 @@ def test_connect_pull_should_debug_log_on_StreamClosedError():
         with patch.object(
             salt.utils.event.log, "debug", auto_spec=True
         ) as mock_log_debug:
-            mock_pusher.connect.side_effect = (
-                salt.ext.tornado.iostream.StreamClosedError
-            )
+            mock_pusher.connect.side_effect = tornado.iostream.StreamClosedError
             event.connect_pull()
             call = mock_log_debug.mock_calls[0]
             assert call.args[0] == "Unable to connect pusher: %s"
-            assert isinstance(call.args[1], salt.ext.tornado.iostream.StreamClosedError)
+            assert isinstance(call.args[1], tornado.iostream.StreamClosedError)
             assert call.args[1].args[0] == "Stream is closed"
 
 
@@ -327,9 +325,7 @@ def test_connect_pull_should_error_log_on_other_errors(error):
                 mock_log_debug.assert_not_called()
                 call = mock_log_error.mock_calls[0]
                 assert call.args[0] == "Unable to connect pusher: %s"
-                assert not isinstance(
-                    call.args[1], salt.ext.tornado.iostream.StreamClosedError
-                )
+                assert not isinstance(call.args[1], tornado.iostream.StreamClosedError)
 
 
 @pytest.mark.slow_test
diff --git a/tests/pytests/unit/utils/event/test_event_return.py b/tests/pytests/unit/utils/event/test_event_return.py
index e9548c701f4..5c4778cbd82 100644
--- a/tests/pytests/unit/utils/event/test_event_return.py
+++ b/tests/pytests/unit/utils/event/test_event_return.py
@@ -1,7 +1,6 @@
 import pytest
 from pytestshellutils.utils.processes import terminate_process
 
-import salt.ext.tornado.ioloop
 import salt.utils.event
 import salt.utils.stringutils
 
diff --git a/tests/support/helpers.py b/tests/support/helpers.py
index 3556e08853b..a08b8c79685 100644
--- a/tests/support/helpers.py
+++ b/tests/support/helpers.py
@@ -33,12 +33,12 @@ import types
 
 import attr
 import pytest
+import tornado.ioloop
+import tornado.web
 from pytestshellutils.exceptions import ProcessFailed
 from pytestshellutils.utils import ports
 from pytestshellutils.utils.processes import ProcessResult
 
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.web
 import salt.utils.files
 import salt.utils.platform
 import salt.utils.pycrypto
@@ -1277,7 +1277,7 @@ def http_basic_auth(login_cb=lambda username, password: False):
     .. code-block:: python
 
         @http_basic_auth(lambda u, p: u == 'foo' and p == 'bar')
-        class AuthenticatedHandler(salt.ext.tornado.web.RequestHandler):
+        class AuthenticatedHandler(tornado.web.RequestHandler):
             pass
     """
 
@@ -1421,9 +1421,7 @@ class Webserver:
 
         self.port = port
         self.wait = wait
-        self.handler = (
-            handler if handler is not None else salt.ext.tornado.web.StaticFileHandler
-        )
+        self.handler = handler if handler is not None else tornado.web.StaticFileHandler
         self.web_root = None
         self.ssl_opts = ssl_opts
 
@@ -1431,16 +1429,14 @@ class Webserver:
         """
         Threading target which stands up the tornado application
         """
-        self.ioloop = salt.ext.tornado.ioloop.IOLoop()
+        self.ioloop = tornado.ioloop.IOLoop()
         self.ioloop.make_current()
-        if self.handler == salt.ext.tornado.web.StaticFileHandler:
-            self.application = salt.ext.tornado.web.Application(
+        if self.handler == tornado.web.StaticFileHandler:
+            self.application = tornado.web.Application(
                 [(r"/(.*)", self.handler, {"path": self.root})]
             )
         else:
-            self.application = salt.ext.tornado.web.Application(
-                [(r"/(.*)", self.handler)]
-            )
+            self.application = tornado.web.Application([(r"/(.*)", self.handler)])
         self.application.listen(self.port, ssl_options=self.ssl_opts)
         self.ioloop.start()
 
@@ -1515,7 +1511,7 @@ class Webserver:
         self.stop()
 
 
-class SaveRequestsPostHandler(salt.ext.tornado.web.RequestHandler):
+class SaveRequestsPostHandler(tornado.web.RequestHandler):
     """
     Save all requests sent to the server.
     """
@@ -1535,7 +1531,7 @@ class SaveRequestsPostHandler(salt.ext.tornado.web.RequestHandler):
         raise NotImplementedError()
 
 
-class MirrorPostHandler(salt.ext.tornado.web.RequestHandler):
+class MirrorPostHandler(tornado.web.RequestHandler):
     """
     Mirror a POST body back to the client
     """
diff --git a/tests/support/netapi.py b/tests/support/netapi.py
index 0df12b0751f..f61650173b9 100644
--- a/tests/support/netapi.py
+++ b/tests/support/netapi.py
@@ -2,14 +2,14 @@ import logging
 import socket
 
 import attr
+import tornado.escape
+import tornado.web
+from tornado import netutil
+from tornado.httpclient import AsyncHTTPClient, HTTPError
+from tornado.httpserver import HTTPServer
+from tornado.ioloop import TimeoutError as IOLoopTimeoutError
 
 import salt.auth
-import salt.ext.tornado.escape
-import salt.ext.tornado.web
-from salt.ext.tornado import netutil
-from salt.ext.tornado.httpclient import AsyncHTTPClient, HTTPError
-from salt.ext.tornado.httpserver import HTTPServer
-from salt.ext.tornado.ioloop import TimeoutError as IOLoopTimeoutError
 from salt.netapi.rest_tornado import saltnado
 
 log = logging.getLogger(__name__)
@@ -46,7 +46,7 @@ class TestsHttpClient:
             if response.headers.get("Content-Type") == "application/json":
                 response._body = response.body.decode("utf-8")
             else:
-                response._body = salt.ext.tornado.escape.native_str(response.body)
+                response._body = tornado.escape.native_str(response.body)
         return response
 
 
@@ -115,7 +115,7 @@ def auth_token(load_auth, auth_creds):
 def build_tornado_app(
     urls, load_auth, client_config, minion_config, setup_event_listener=False
 ):
-    application = salt.ext.tornado.web.Application(urls, debug=True)
+    application = tornado.web.Application(urls, debug=True)
 
     application.auth = load_auth
     application.opts = client_config
diff --git a/tests/support/pytest/transport.py b/tests/support/pytest/transport.py
index eaa8adc8bd4..038c8a6cde6 100644
--- a/tests/support/pytest/transport.py
+++ b/tests/support/pytest/transport.py
@@ -5,14 +5,14 @@ import socket
 import time
 
 import pytest
+import tornado.gen
+import tornado.ioloop
+import tornado.iostream
 import zmq
 from pytestshellutils.utils.processes import terminate_process
 
 import salt.channel.server
 import salt.exceptions
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.iostream
 import salt.master
 import salt.utils.msgpack
 import salt.utils.process
@@ -93,9 +93,9 @@ class Collector(salt.utils.process.SignalHandlingProcess):
                     time.sleep(1)
                 else:
                     break
-            self.sock = salt.ext.tornado.iostream.IOStream(sock)
+            self.sock = tornado.iostream.IOStream(sock)
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _recv(self):
         if self.transport == "zeromq":
             # test_zeromq_filtering requires catching the
@@ -103,19 +103,19 @@ class Collector(salt.utils.process.SignalHandlingProcess):
             try:
                 payload = self.sock.recv(zmq.NOBLOCK)
                 serial_payload = salt.payload.loads(payload)
-                raise salt.ext.tornado.gen.Return(serial_payload)
+                raise tornado.gen.Return(serial_payload)
             except (zmq.ZMQError, salt.exceptions.SaltDeserializationError):
                 raise RecvError("ZMQ Error")
         else:
             for msg in self.unpacker:
-                raise salt.ext.tornado.gen.Return(msg["body"])
+                raise tornado.gen.Return(msg["body"])
             byts = yield self.sock.read_bytes(8096, partial=True)
             self.unpacker.feed(byts)
             for msg in self.unpacker:
-                raise salt.ext.tornado.gen.Return(msg["body"])
+                raise tornado.gen.Return(msg["body"])
             raise RecvError("TCP Error")
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def _run(self, loop):
         try:
             self._setup_listener()
@@ -165,7 +165,7 @@ class Collector(salt.utils.process.SignalHandlingProcess):
         Gather results until then number of seconds specified by timeout passes
         without receiving a message
         """
-        loop = salt.ext.tornado.ioloop.IOLoop()
+        loop = tornado.ioloop.IOLoop()
         loop.add_callback(self._run, loop)
         loop.start()
 
diff --git a/tests/unit/modules/test_random_org.py b/tests/unit/modules/test_random_org.py
index 30e98ca0802..1d4db7bf46c 100644
--- a/tests/unit/modules/test_random_org.py
+++ b/tests/unit/modules/test_random_org.py
@@ -3,9 +3,9 @@
 """
 
 import pytest
+from tornado.httpclient import HTTPClient
 
 import salt.modules.random_org as random_org
-from salt.ext.tornado.httpclient import HTTPClient
 from tests.support.mixins import LoaderModuleMockMixin
 from tests.support.unit import TestCase
 
diff --git a/tests/unit/netapi/rest_tornado/test_saltnado.py b/tests/unit/netapi/rest_tornado/test_saltnado.py
index 7b63a65d4f3..0873ef3af34 100644
--- a/tests/unit/netapi/rest_tornado/test_saltnado.py
+++ b/tests/unit/netapi/rest_tornado/test_saltnado.py
@@ -1,10 +1,11 @@
-import salt.ext.tornado
-import salt.ext.tornado.testing
+import tornado
+import tornado.testing
+
 import salt.netapi.rest_tornado.saltnado as saltnado
 from tests.support.mock import MagicMock, patch
 
 
-class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
+class TestJobNotRunning(tornado.testing.AsyncTestCase):
     def setUp(self):
         super().setUp()
         self.mock = MagicMock()
@@ -23,11 +24,11 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
         self.handler.lowstate = []
         self.handler.content_type = "text/plain"
         self.handler.dumper = lambda x: x
-        f = salt.ext.tornado.gen.Future()
+        f = tornado.gen.Future()
         f.set_result({"jid": f, "minions": []})
         self.handler.saltclients.update({"local": lambda *args, **kwargs: f})
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_disbatch_has_already_finished_then_writing_return_should_not_fail(
         self,
     ):
@@ -37,7 +38,7 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
         # Asserting that it doesn't raise anything is... the default behavior
         # for a test.
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_disbatch_has_already_finished_then_finishing_should_not_fail(self):
         self.handler.finish()
         result = yield self.handler.disbatch()
@@ -45,12 +46,12 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
         # Asserting that it doesn't raise anything is... the default behavior
         # for a test.
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_event_times_out_and_minion_is_not_running_result_should_be_True(self):
-        fut = salt.ext.tornado.gen.Future()
+        fut = tornado.gen.Future()
         fut.set_exception(saltnado.TimeoutException())
         self.mock.event_listener.get_event.return_value = fut
-        wrong_future = salt.ext.tornado.gen.Future()
+        wrong_future = tornado.gen.Future()
 
         result = yield self.handler.job_not_running(
             jid=42, tgt="*", tgt_type="glob", minions=[], is_finished=wrong_future
@@ -58,14 +59,14 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
 
         self.assertTrue(result)
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_event_times_out_and_minion_is_not_running_minion_data_should_not_be_set(
         self,
     ):
-        fut = salt.ext.tornado.gen.Future()
+        fut = tornado.gen.Future()
         fut.set_exception(saltnado.TimeoutException())
         self.mock.event_listener.get_event.return_value = fut
-        wrong_future = salt.ext.tornado.gen.Future()
+        wrong_future = tornado.gen.Future()
         minions = {}
 
         result = yield self.handler.job_not_running(
@@ -74,20 +75,20 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
 
         assert not minions
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_event_finally_finishes_and_returned_minion_not_in_minions_it_should_be_set_to_False(
         self,
     ):
         expected_id = 42
-        no_data_event = salt.ext.tornado.gen.Future()
+        no_data_event = tornado.gen.Future()
         no_data_event.set_result({"data": {}})
-        empty_return_event = salt.ext.tornado.gen.Future()
+        empty_return_event = tornado.gen.Future()
         empty_return_event.set_result({"data": {"return": {}}})
-        actual_return_event = salt.ext.tornado.gen.Future()
+        actual_return_event = tornado.gen.Future()
         actual_return_event.set_result(
             {"data": {"return": {"something happened here": "OK?"}, "id": expected_id}}
         )
-        timed_out_event = salt.ext.tornado.gen.Future()
+        timed_out_event = tornado.gen.Future()
         timed_out_event.set_exception(saltnado.TimeoutException())
         self.mock.event_listener.get_event.side_effect = [
             no_data_event,
@@ -103,27 +104,27 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
             tgt="*",
             tgt_type="fnord",
             minions=minions,
-            is_finished=salt.ext.tornado.gen.Future(),
+            is_finished=tornado.gen.Future(),
         )
 
         self.assertFalse(minions[expected_id])
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_event_finally_finishes_and_returned_minion_already_in_minions_it_should_not_be_changed(
         self,
     ):
         expected_id = 42
         expected_value = object()
         minions = {expected_id: expected_value}
-        no_data_event = salt.ext.tornado.gen.Future()
+        no_data_event = tornado.gen.Future()
         no_data_event.set_result({"data": {}})
-        empty_return_event = salt.ext.tornado.gen.Future()
+        empty_return_event = tornado.gen.Future()
         empty_return_event.set_result({"data": {"return": {}}})
-        actual_return_event = salt.ext.tornado.gen.Future()
+        actual_return_event = tornado.gen.Future()
         actual_return_event.set_result(
             {"data": {"return": {"something happened here": "OK?"}, "id": expected_id}}
         )
-        timed_out_event = salt.ext.tornado.gen.Future()
+        timed_out_event = tornado.gen.Future()
         timed_out_event.set_exception(saltnado.TimeoutException())
         self.mock.event_listener.get_event.side_effect = [
             no_data_event,
@@ -138,22 +139,22 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
             tgt="*",
             tgt_type="fnord",
             minions=minions,
-            is_finished=salt.ext.tornado.gen.Future(),
+            is_finished=tornado.gen.Future(),
         )
 
         self.assertIs(minions[expected_id], expected_value)
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_event_returns_early_and_finally_times_out_result_should_be_True(self):
-        no_data_event = salt.ext.tornado.gen.Future()
+        no_data_event = tornado.gen.Future()
         no_data_event.set_result({"data": {}})
-        empty_return_event = salt.ext.tornado.gen.Future()
+        empty_return_event = tornado.gen.Future()
         empty_return_event.set_result({"data": {"return": {}}})
-        actual_return_event = salt.ext.tornado.gen.Future()
+        actual_return_event = tornado.gen.Future()
         actual_return_event.set_result(
             {"data": {"return": {"something happened here": "OK?"}, "id": "fnord"}}
         )
-        timed_out_event = salt.ext.tornado.gen.Future()
+        timed_out_event = tornado.gen.Future()
         timed_out_event.set_exception(saltnado.TimeoutException())
         self.mock.event_listener.get_event.side_effect = [
             no_data_event,
@@ -168,21 +169,21 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
             tgt="*",
             tgt_type="fnord",
             minions={},
-            is_finished=salt.ext.tornado.gen.Future(),
+            is_finished=tornado.gen.Future(),
         )
         self.assertTrue(result)
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_event_finishes_but_is_finished_is_done_then_result_should_be_True(
         self,
     ):
         expected_minion_id = "fnord"
         expected_minion_value = object()
-        no_data_event = salt.ext.tornado.gen.Future()
+        no_data_event = tornado.gen.Future()
         no_data_event.set_result({"data": {}})
-        empty_return_event = salt.ext.tornado.gen.Future()
+        empty_return_event = tornado.gen.Future()
         empty_return_event.set_result({"data": {"return": {}}})
-        actual_return_event = salt.ext.tornado.gen.Future()
+        actual_return_event = tornado.gen.Future()
         actual_return_event.set_result(
             {
                 "data": {
@@ -191,11 +192,11 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
                 }
             }
         )
-        is_finished = salt.ext.tornado.gen.Future()
+        is_finished = tornado.gen.Future()
 
         def abort(*args, **kwargs):
             yield actual_return_event
-            f = salt.ext.tornado.gen.Future()
+            f = tornado.gen.Future()
             f.set_exception(saltnado.TimeoutException())
             is_finished.set_result("This is done")
             yield f
@@ -218,14 +219,14 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
         self.assertTrue(len(minions) == 1, str(minions))
         self.assertIs(minions[expected_minion_id], expected_minion_value)
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_is_finished_times_out_before_event_finishes_result_should_be_True(
         self,
     ):
         # Other test times out with event - this one should time out for is_finished
-        finished = salt.ext.tornado.gen.Future()
+        finished = tornado.gen.Future()
         finished.set_exception(saltnado.TimeoutException())
-        wrong_future = salt.ext.tornado.gen.Future()
+        wrong_future = tornado.gen.Future()
         self.mock.event_listener.get_event.return_value = wrong_future
 
         result = yield self.handler.job_not_running(
@@ -234,13 +235,13 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
 
         self.assertTrue(result)
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_is_finished_times_out_before_event_finishes_event_should_have_result_set_to_None(
         self,
     ):
-        finished = salt.ext.tornado.gen.Future()
+        finished = tornado.gen.Future()
         finished.set_exception(saltnado.TimeoutException())
-        wrong_future = salt.ext.tornado.gen.Future()
+        wrong_future = tornado.gen.Future()
         self.mock.event_listener.get_event.return_value = wrong_future
 
         result = yield self.handler.job_not_running(
@@ -251,7 +252,7 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
 
 
 # TODO: I think we can extract seUp into a superclass -W. Werner, 2020-11-03
-class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
+class TestGetMinionReturns(tornado.testing.AsyncTestCase):
     def setUp(self):
         super().setUp()
         self.mock = MagicMock()
@@ -265,22 +266,22 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
             "gather_job_timeout": 10.001,
         }
         self.handler = saltnado.SaltAPIHandler(self.mock, self.mock)
-        f = salt.ext.tornado.gen.Future()
+        f = tornado.gen.Future()
         f.set_result({"jid": f, "minions": []})
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_if_finished_before_any_events_return_then_result_should_be_empty_dictionary(
         self,
     ):
         expected_result = {}
-        xxx = salt.ext.tornado.gen.Future()
+        xxx = tornado.gen.Future()
         xxx.set_result(None)
-        is_finished = salt.ext.tornado.gen.Future()
+        is_finished = tornado.gen.Future()
         is_finished.set_result(None)
         actual_result = yield self.handler.get_minion_returns(
             events=[],
             is_finished=is_finished,
-            is_timed_out=salt.ext.tornado.gen.Future(),
+            is_timed_out=tornado.gen.Future(),
             min_wait_time=xxx,
             minions={},
         )
@@ -288,7 +289,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
 
     # TODO: Copy above - test with timed out -W. Werner, 2020-11-05
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_if_is_finished_after_events_return_then_result_should_contain_event_result_data(
         self,
     ):
@@ -296,15 +297,15 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
             "minion1": {"fnord": "this is some fnordish data"},
             "minion2": {"fnord": "this is some other fnordish data"},
         }
-        xxx = salt.ext.tornado.gen.Future()
+        xxx = tornado.gen.Future()
         xxx.set_result(None)
-        is_finished = salt.ext.tornado.gen.Future()
+        is_finished = tornado.gen.Future()
         # XXX what do I do here?
         events = [
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
         ]
         events[0].set_result(
             {
@@ -323,7 +324,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
         actual_result = yield self.handler.get_minion_returns(
             events=events,
             is_finished=is_finished,
-            is_timed_out=salt.ext.tornado.gen.Future(),
+            is_timed_out=tornado.gen.Future(),
             min_wait_time=xxx,
             minions={
                 "minion1": False,
@@ -334,7 +335,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
 
         assert actual_result == expected_result
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_if_timed_out_after_events_return_then_result_should_contain_event_result_data(
         self,
     ):
@@ -342,15 +343,15 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
             "minion1": {"fnord": "this is some fnordish data"},
             "minion2": {"fnord": "this is some other fnordish data"},
         }
-        xxx = salt.ext.tornado.gen.Future()
+        xxx = tornado.gen.Future()
         xxx.set_result(None)
-        is_timed_out = salt.ext.tornado.gen.Future()
+        is_timed_out = tornado.gen.Future()
         # XXX what do I do here?
         events = [
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
         ]
         events[0].set_result(
             {
@@ -368,7 +369,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
 
         actual_result = yield self.handler.get_minion_returns(
             events=events,
-            is_finished=salt.ext.tornado.gen.Future(),
+            is_finished=tornado.gen.Future(),
             is_timed_out=is_timed_out,
             min_wait_time=xxx,
             minions={
@@ -380,7 +381,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
 
         assert actual_result == expected_result
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_if_wait_timer_is_not_done_even_though_results_are_then_data_should_not_yet_be_returned(
         self,
     ):
@@ -388,18 +389,18 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
             "one": {"fnordy one": "one has some data"},
             "two": {"fnordy two": "two has some data"},
         }
-        events = [salt.ext.tornado.gen.Future(), salt.ext.tornado.gen.Future()]
+        events = [tornado.gen.Future(), tornado.gen.Future()]
         events[0].set_result(
             {"tag": "fnord", "data": {"id": "one", "return": expected_result["one"]}}
         )
         events[1].set_result(
             {"tag": "fnord", "data": {"id": "two", "return": expected_result["two"]}}
         )
-        wait_timer = salt.ext.tornado.gen.Future()
+        wait_timer = tornado.gen.Future()
         fut = self.handler.get_minion_returns(
             events=events,
-            is_finished=salt.ext.tornado.gen.Future(),
-            is_timed_out=salt.ext.tornado.gen.Future(),
+            is_finished=tornado.gen.Future(),
+            is_timed_out=tornado.gen.Future(),
             min_wait_time=wait_timer,
             minions={"one": False, "two": False},
         )
@@ -408,7 +409,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
             yield fut
 
         self.io_loop.spawn_callback(boop)
-        yield salt.ext.tornado.gen.sleep(0.1)
+        yield tornado.gen.sleep(0.1)
 
         assert not fut.done()
 
@@ -417,30 +418,30 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
 
         assert actual_result == expected_result
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_is_finished_any_other_futures_should_be_canceled(self):
         events = [
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
         ]
 
-        is_finished = salt.ext.tornado.gen.Future()
+        is_finished = tornado.gen.Future()
         is_finished.set_result(None)
         yield self.handler.get_minion_returns(
             events=events,
             is_finished=is_finished,
-            is_timed_out=salt.ext.tornado.gen.Future(),
-            min_wait_time=salt.ext.tornado.gen.Future(),
+            is_timed_out=tornado.gen.Future(),
+            min_wait_time=tornado.gen.Future(),
             minions={"one": False, "two": False},
         )
 
         are_done = [event.done() for event in events]
         assert all(are_done)
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_an_event_times_out_then_we_should_not_enter_an_infinite_loop(self):
         # NOTE: this test will enter an infinite loop if the code is broken. I
         # was not able to figure out a way to ensure that the test exits with
@@ -451,27 +452,27 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
         # TimeoutException.
 
         events = [
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
         ]
 
         # Arguably any event would work, but 3 isn't the first, so it
         # gives us a little more confidence that this test is testing
         # correctly
         events[3].set_exception(saltnado.TimeoutException())
-        times_out_later = salt.ext.tornado.gen.Future()
+        times_out_later = tornado.gen.Future()
         # 0.5s should be long enough that the test gets through doing other
         # things before hitting this timeout, which will cancel all the
         # in-flight futures.
         self.io_loop.call_later(0.5, lambda: times_out_later.set_result(None))
         yield self.handler.get_minion_returns(
             events=events,
-            is_finished=salt.ext.tornado.gen.Future(),
+            is_finished=tornado.gen.Future(),
             is_timed_out=times_out_later,
-            min_wait_time=salt.ext.tornado.gen.Future(),
+            min_wait_time=tornado.gen.Future(),
             minions={"one": False, "two": False},
         )
 
@@ -482,7 +483,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
         assert all(are_done)
         assert times_out_later.done()
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_is_timed_out_any_other_futures_should_be_canceled(self):
         # There is some question about whether this test is or should be
         # necessary. Or if it's meaningful. The code that this is testing
@@ -491,46 +492,46 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
         # That being said, the worst case is that this is just a duplicate
         # or irrelevant test, and can be removed.
         events = [
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
         ]
 
-        is_timed_out = salt.ext.tornado.gen.Future()
+        is_timed_out = tornado.gen.Future()
         is_timed_out.set_result(None)
         yield self.handler.get_minion_returns(
             events=events,
-            is_finished=salt.ext.tornado.gen.Future(),
+            is_finished=tornado.gen.Future(),
             is_timed_out=is_timed_out,
-            min_wait_time=salt.ext.tornado.gen.Future(),
+            min_wait_time=tornado.gen.Future(),
             minions={"one": False, "two": False},
         )
 
         are_done = [event.done() for event in events]
         assert all(are_done)
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_min_wait_time_and_nothing_todo_any_other_futures_should_be_canceled(
         self,
     ):
         events = [
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
-            salt.ext.tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
+            tornado.gen.Future(),
         ]
 
-        is_finished = salt.ext.tornado.gen.Future()
-        min_wait_time = salt.ext.tornado.gen.Future()
+        is_finished = tornado.gen.Future()
+        min_wait_time = tornado.gen.Future()
         self.io_loop.call_later(0.2, lambda: min_wait_time.set_result(None))
 
         yield self.handler.get_minion_returns(
             events=events,
             is_finished=is_finished,
-            is_timed_out=salt.ext.tornado.gen.Future(),
+            is_timed_out=tornado.gen.Future(),
             min_wait_time=min_wait_time,
             minions={"one": True, "two": True},
         )
@@ -538,37 +539,37 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
         are_done = [event.done() for event in events] + [is_finished.done()]
         assert all(are_done)
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_is_finished_but_not_is_timed_out_then_timed_out_should_not_be_set_to_done(
         self,
     ):
-        events = [salt.ext.tornado.gen.Future()]
-        is_timed_out = salt.ext.tornado.gen.Future()
-        is_finished = salt.ext.tornado.gen.Future()
+        events = [tornado.gen.Future()]
+        is_timed_out = tornado.gen.Future()
+        is_finished = tornado.gen.Future()
         is_finished.set_result(None)
 
         yield self.handler.get_minion_returns(
             events=events,
             is_finished=is_finished,
             is_timed_out=is_timed_out,
-            min_wait_time=salt.ext.tornado.gen.Future(),
+            min_wait_time=tornado.gen.Future(),
             minions={"one": False, "two": False},
         )
 
         assert not is_timed_out.done()
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_min_wait_time_and_all_completed_but_not_is_timed_out_then_timed_out_should_not_be_set_to_done(
         self,
     ):
-        events = [salt.ext.tornado.gen.Future()]
-        is_timed_out = salt.ext.tornado.gen.Future()
-        min_wait_time = salt.ext.tornado.gen.Future()
+        events = [tornado.gen.Future()]
+        is_timed_out = tornado.gen.Future()
+        min_wait_time = tornado.gen.Future()
         self.io_loop.call_later(0.2, lambda: min_wait_time.set_result(None))
 
         yield self.handler.get_minion_returns(
             events=events,
-            is_finished=salt.ext.tornado.gen.Future(),
+            is_finished=tornado.gen.Future(),
             is_timed_out=is_timed_out,
             min_wait_time=min_wait_time,
             minions={"one": True},
@@ -576,21 +577,21 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
 
         assert not is_timed_out.done()
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_things_are_completed_but_not_timed_out_then_timed_out_event_should_not_be_done(
         self,
     ):
         events = [
-            salt.ext.tornado.gen.Future(),
+            tornado.gen.Future(),
         ]
         events[0].set_result({"tag": "fnord", "data": {"id": "one", "return": {}}})
-        min_wait_time = salt.ext.tornado.gen.Future()
+        min_wait_time = tornado.gen.Future()
         min_wait_time.set_result(None)
-        is_timed_out = salt.ext.tornado.gen.Future()
+        is_timed_out = tornado.gen.Future()
 
         yield self.handler.get_minion_returns(
             events=events,
-            is_finished=salt.ext.tornado.gen.Future(),
+            is_finished=tornado.gen.Future(),
             is_timed_out=is_timed_out,
             min_wait_time=min_wait_time,
             minions={"one": True},
@@ -599,7 +600,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
         assert not is_timed_out.done()
 
 
-class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
+class TestDisbatchLocal(tornado.testing.AsyncTestCase):
     def setUp(self):
         super().setUp()
         self.mock = MagicMock()
@@ -614,12 +615,12 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
         }
         self.handler = saltnado.SaltAPIHandler(self.mock, self.mock)
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_is_timed_out_is_set_before_other_events_are_completed_then_result_should_be_empty_dictionary(
         self,
     ):
-        completed_event = salt.ext.tornado.gen.Future()
-        never_completed = salt.ext.tornado.gen.Future()
+        completed_event = tornado.gen.Future()
+        never_completed = tornado.gen.Future()
         # TODO: We may need to tweak these values to get them close enough but not so far away -W. Werner, 2020-11-17
         gather_timeout = 0.1
         event_timeout = gather_timeout + 0.05
@@ -642,7 +643,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
 
         self.io_loop.call_later(event_timeout, completer)
 
-        f = salt.ext.tornado.gen.Future()
+        f = tornado.gen.Future()
         f.set_result({"jid": "42", "minions": []})
         with patch.object(
             self.handler.application.event_listener,
@@ -661,12 +662,12 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
 
         assert result == {}
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_is_finished_is_set_before_events_return_then_no_data_should_be_returned(
         self,
     ):
-        completed_event = salt.ext.tornado.gen.Future()
-        never_completed = salt.ext.tornado.gen.Future()
+        completed_event = tornado.gen.Future()
+        never_completed = tornado.gen.Future()
         gather_timeout = 2
         event_timeout = gather_timeout - 1
 
@@ -693,7 +694,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
             assert finished is not None
             finished.set_result(42)
 
-        f = salt.ext.tornado.gen.Future()
+        f = tornado.gen.Future()
         f.set_result({"jid": "42", "minions": []})
         with patch.object(
             self.handler.application.event_listener,
@@ -717,13 +718,13 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
 
         assert result == {}
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_is_finished_then_all_collected_data_should_be_returned(self):
-        completed_event = salt.ext.tornado.gen.Future()
-        never_completed = salt.ext.tornado.gen.Future()
+        completed_event = tornado.gen.Future()
+        never_completed = tornado.gen.Future()
         # This timeout should never be reached
         gather_timeout = 42
-        completed_events = [salt.ext.tornado.gen.Future() for _ in range(5)]
+        completed_events = [tornado.gen.Future() for _ in range(5)]
         for i, event in enumerate(completed_events):
             event.set_result(
                 {
@@ -734,7 +735,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
                     },
                 }
             )
-        uncompleted_events = [salt.ext.tornado.gen.Future() for _ in range(5)]
+        uncompleted_events = [tornado.gen.Future() for _ in range(5)]
         events = iter(completed_events + uncompleted_events)
         expected_result = {
             "fnord 0": "return from fnord 0",
@@ -755,7 +756,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
             assert finished is not None
             finished.set_result(42)
 
-        f = salt.ext.tornado.gen.Future()
+        f = tornado.gen.Future()
         f.set_result({"jid": "42", "minions": ["non-existent minion"]})
         with patch.object(
             self.handler.application.event_listener,
@@ -779,16 +780,16 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
 
         assert result == expected_result
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_is_timed_out_then_all_collected_data_should_be_returned(self):
-        completed_event = salt.ext.tornado.gen.Future()
-        never_completed = salt.ext.tornado.gen.Future()
+        completed_event = tornado.gen.Future()
+        never_completed = tornado.gen.Future()
         # 2s is probably enough for any kind of computer to manage to
         # do all the other processing. We could maybe reduce this - just
         # depends on how slow of a system we're running on.
         # TODO: Maybe we should have a test helper/fixture that benchmarks the system and gets a reasonable timeout? -W. Werner, 2020-11-19
         gather_timeout = 2
-        completed_events = [salt.ext.tornado.gen.Future() for _ in range(5)]
+        completed_events = [tornado.gen.Future() for _ in range(5)]
         for i, event in enumerate(completed_events):
             event.set_result(
                 {
@@ -799,7 +800,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
                     },
                 }
             )
-        uncompleted_events = [salt.ext.tornado.gen.Future() for _ in range(5)]
+        uncompleted_events = [tornado.gen.Future() for _ in range(5)]
         events = iter(completed_events + uncompleted_events)
         expected_result = {
             "fnord 0": "return from fnord 0",
@@ -815,7 +816,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
             else:
                 return next(events)
 
-        f = salt.ext.tornado.gen.Future()
+        f = tornado.gen.Future()
         f.set_result({"jid": "42", "minions": ["non-existent minion"]})
         with patch.object(
             self.handler.application.event_listener,
@@ -834,13 +835,13 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
 
         assert result == expected_result
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_minions_all_return_then_all_collected_data_should_be_returned(self):
-        completed_event = salt.ext.tornado.gen.Future()
-        never_completed = salt.ext.tornado.gen.Future()
+        completed_event = tornado.gen.Future()
+        never_completed = tornado.gen.Future()
         # Timeout is something ridiculously high - it should never be reached
         gather_timeout = 20
-        completed_events = [salt.ext.tornado.gen.Future() for _ in range(10)]
+        completed_events = [tornado.gen.Future() for _ in range(10)]
         events_by_id = {}
         for i, event in enumerate(completed_events):
             id_ = "fnord {}".format(i)
@@ -868,7 +869,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
             tag = kwargs.get("tag", "").rpartition("/")[-1]
             return events_by_id.get(tag, never_completed)
 
-        f = salt.ext.tornado.gen.Future()
+        f = tornado.gen.Future()
         f.set_result(
             {
                 "jid": "42",
@@ -892,15 +893,15 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
 
         assert result == expected_result
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_when_min_wait_time_has_not_passed_then_disbatch_should_not_return_expected_data_until_time_has_passed(
         self,
     ):
-        completed_event = salt.ext.tornado.gen.Future()
-        never_completed = salt.ext.tornado.gen.Future()
-        wait_timer = salt.ext.tornado.gen.Future()
+        completed_event = tornado.gen.Future()
+        never_completed = tornado.gen.Future()
+        wait_timer = tornado.gen.Future()
         gather_timeout = 20
-        completed_events = [salt.ext.tornado.gen.Future() for _ in range(10)]
+        completed_events = [tornado.gen.Future() for _ in range(10)]
         events_by_id = {}
         # Setup some real-enough looking return data
         for i, event in enumerate(completed_events):
@@ -948,11 +949,11 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
         # The fake sleep is necessary so that we can return our own
         # min_wait_time future. The fakeo_timer object is how we signal
         # which one we need to be returning.
-        orig_sleep = salt.ext.tornado.gen.sleep
+        orig_sleep = tornado.gen.sleep
 
         fakeo_timer = object()
 
-        @salt.ext.tornado.gen.coroutine
+        @tornado.gen.coroutine
         def fake_sleep(timer):
             # only return our fake min_wait_time future when the sentinel
             # value is provided. Otherwise it's just a number.
@@ -961,7 +962,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
             else:
                 yield orig_sleep(timer)
 
-        f = salt.ext.tornado.gen.Future()
+        f = tornado.gen.Future()
         f.set_result(
             {
                 "jid": "42",
@@ -987,7 +988,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
                 "order_masters": True,
             },
         ), patch(
-            "salt.ext.tornado.gen.sleep",
+            "tornado.gen.sleep",
             autospec=True,
             side_effect=fake_sleep,
         ), patch.dict(
@@ -1015,7 +1016,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
                 yield fut
 
             self.io_loop.spawn_callback(boop)
-            yield salt.ext.tornado.gen.sleep(0.1)
+            yield tornado.gen.sleep(0.1)
             # here, all the minions should be complete (i.e. "True")
             assert all(minions[m_id] for m_id in minions)
             # But _disbatch_local is not returned yet because min_wait_time has not passed
diff --git a/tests/unit/test_proxy_minion.py b/tests/unit/test_proxy_minion.py
index bc3e867619f..92241cdef71 100644
--- a/tests/unit/test_proxy_minion.py
+++ b/tests/unit/test_proxy_minion.py
@@ -10,11 +10,11 @@ import tempfile
 import textwrap
 
 import pytest
+import tornado
+import tornado.testing
 from saltfactories.utils import random_string
 
 import salt.config
-import salt.ext.tornado
-import salt.ext.tornado.testing
 import salt.metaproxy.proxy
 import salt.minion
 import salt.syspaths
@@ -38,7 +38,7 @@ class ProxyMinionTestCase(TestCase):
         proxy_minion = salt.minion.ProxyMinion(
             mock_opts,
             jid_queue=copy.copy(mock_jid_queue),
-            io_loop=salt.ext.tornado.ioloop.IOLoop(),
+            io_loop=tornado.ioloop.IOLoop(),
         )
         mock_metaproxy_call = MagicMock()
         with patch(
@@ -65,7 +65,7 @@ class ProxyMinionTestCase(TestCase):
         proxy_minion = salt.minion.ProxyMinion(
             mock_opts,
             jid_queue=copy.copy(mock_jid_queue),
-            io_loop=salt.ext.tornado.ioloop.IOLoop(),
+            io_loop=tornado.ioloop.IOLoop(),
         )
         mock_metaproxy_call = MagicMock()
         with patch(
@@ -93,7 +93,7 @@ class ProxyMinionTestCase(TestCase):
         proxy_minion = salt.minion.ProxyMinion(
             mock_opts,
             jid_queue=copy.copy(mock_jid_queue),
-            io_loop=salt.ext.tornado.ioloop.IOLoop(),
+            io_loop=tornado.ioloop.IOLoop(),
         )
         mock_metaproxy_call = MagicMock()
         with patch(
diff --git a/tests/unit/transport/mixins.py b/tests/unit/transport/mixins.py
index 65b1c7a9498..c757f37861f 100644
--- a/tests/unit/transport/mixins.py
+++ b/tests/unit/transport/mixins.py
@@ -1,4 +1,4 @@
-import salt.ext.tornado.gen
+import tornado.gen
 
 
 def run_loop_in_thread(loop, evt):
@@ -7,13 +7,13 @@ def run_loop_in_thread(loop, evt):
     """
     loop.make_current()
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def stopper():
         while True:
             if evt.is_set():
                 loop.stop()
                 break
-            yield salt.ext.tornado.gen.sleep(0.3)
+            yield tornado.gen.sleep(0.3)
 
     loop.add_callback(stopper)
     try:
diff --git a/tests/unit/transport/test_ipc.py b/tests/unit/transport/test_ipc.py
index acc01cd705d..639a9f606f6 100644
--- a/tests/unit/transport/test_ipc.py
+++ b/tests/unit/transport/test_ipc.py
@@ -7,15 +7,15 @@ import os
 import threading
 
 import pytest
+import tornado.gen
+import tornado.ioloop
+import tornado.testing
+from tornado.iostream import StreamClosedError
 
 import salt.config
 import salt.exceptions
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.testing
 import salt.transport.ipc
 import salt.utils.platform
-from salt.ext.tornado.iostream import StreamClosedError
 from tests.support.runtests import RUNTIME_VARS
 
 pytestmark = [
@@ -28,7 +28,7 @@ log = logging.getLogger(__name__)
 
 
 @pytest.mark.skip_on_windows(reason="Windows does not support Posix IPC")
-class IPCMessagePubSubCase(salt.ext.tornado.testing.AsyncTestCase):
+class IPCMessagePubSubCase(tornado.testing.AsyncTestCase):
     """
     Test all of the clear msg stuff
     """
@@ -128,7 +128,7 @@ class IPCMessagePubSubCase(salt.ext.tornado.testing.AsyncTestCase):
         self.assertEqual(ret1, "TEST")
         self.assertEqual(ret2, "TEST")
 
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_async_reading_streamclosederror(self):
         client1 = self.sub_channel
         call_cnt = []
diff --git a/tests/unit/transport/test_tcp.py b/tests/unit/transport/test_tcp.py
index dbe88081859..2719ab02a0c 100644
--- a/tests/unit/transport/test_tcp.py
+++ b/tests/unit/transport/test_tcp.py
@@ -6,18 +6,18 @@ import logging
 import threading
 
 import pytest
+import tornado.concurrent
+import tornado.gen
+import tornado.ioloop
 from pytestshellutils.utils import ports
+from tornado.testing import AsyncTestCase
 
 import salt.channel.client
 import salt.channel.server
 import salt.config
 import salt.exceptions
-import salt.ext.tornado.concurrent
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
 import salt.utils.platform
 import salt.utils.process
-from salt.ext.tornado.testing import AsyncTestCase
 from tests.support.mixins import AdaptedConfigurationTestCaseMixin
 from tests.unit.transport.mixins import run_loop_in_thread
 
@@ -82,7 +82,7 @@ class AsyncPubServerTest(AsyncTestCase, AdaptedConfigurationTestCaseMixin):
             cls.master_config
         )
         cls.req_server_channel.pre_fork(cls.process_manager)
-        cls.io_loop = salt.ext.tornado.ioloop.IOLoop()
+        cls.io_loop = tornado.ioloop.IOLoop()
         cls.stop = threading.Event()
         cls.req_server_channel.post_fork(cls._handle_payload, io_loop=cls.io_loop)
         cls.server_thread = threading.Thread(
diff --git a/tests/unit/utils/test_asynchronous.py b/tests/unit/utils/test_asynchronous.py
index e5bd974cb62..92b37c76ff9 100644
--- a/tests/unit/utils/test_asynchronous.py
+++ b/tests/unit/utils/test_asynchronous.py
@@ -1,7 +1,8 @@
-import salt.ext.tornado.gen
-import salt.ext.tornado.testing
+import tornado.gen
+import tornado.testing
+from tornado.testing import AsyncTestCase
+
 import salt.utils.asynchronous as asynchronous
-from salt.ext.tornado.testing import AsyncTestCase
 
 
 class HelperA:
@@ -13,10 +14,10 @@ class HelperA:
     def __init__(self, io_loop=None):
         pass
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def sleep(self):
-        yield salt.ext.tornado.gen.sleep(0.1)
-        raise salt.ext.tornado.gen.Return(True)
+        yield tornado.gen.sleep(0.1)
+        raise tornado.gen.Return(True)
 
 
 class HelperB:
@@ -30,15 +31,15 @@ class HelperB:
             a = asynchronous.SyncWrapper(HelperA)
         self.a = a
 
-    @salt.ext.tornado.gen.coroutine
+    @tornado.gen.coroutine
     def sleep(self):
-        yield salt.ext.tornado.gen.sleep(0.1)
+        yield tornado.gen.sleep(0.1)
         self.a.sleep()
-        raise salt.ext.tornado.gen.Return(False)
+        raise tornado.gen.Return(False)
 
 
 class TestSyncWrapper(AsyncTestCase):
-    @salt.ext.tornado.testing.gen_test
+    @tornado.testing.gen_test
     def test_helpers(self):
         """
         Test that the helper classes do what we expect within a regular asynchronous env
diff --git a/tests/unit/utils/test_context.py b/tests/unit/utils/test_context.py
index e53bc764fbf..c8b0e5f44f1 100644
--- a/tests/unit/utils/test_context.py
+++ b/tests/unit/utils/test_context.py
@@ -4,7 +4,6 @@ tests.unit.context_test
 """
 
 
-import salt.ext.tornado.gen
 import salt.utils.json
 from salt.utils.context import NamespacedDictWrapper
 from tests.support.unit import TestCase
diff --git a/tests/unit/utils/test_gitfs.py b/tests/unit/utils/test_gitfs.py
index b99da3ef916..55e25eaf0d2 100644
--- a/tests/unit/utils/test_gitfs.py
+++ b/tests/unit/utils/test_gitfs.py
@@ -7,6 +7,7 @@ import shutil
 from time import time
 
 import pytest
+import tornado.ioloop
 
 import salt.fileserver.gitfs
 import salt.utils.files
@@ -35,9 +36,7 @@ if HAS_PYGIT2:
 
 def _clear_instance_map():
     try:
-        del salt.utils.gitfs.GitFS.instance_map[
-            salt.ext.tornado.ioloop.IOLoop.current()
-        ]
+        del salt.utils.gitfs.GitFS.instance_map[tornado.ioloop.IOLoop.current()]
     except KeyError:
         pass
 
diff --git a/tests/unit/utils/test_http.py b/tests/unit/utils/test_http.py
index 9f7a60ffa73..0a2d24d97b5 100644
--- a/tests/unit/utils/test_http.py
+++ b/tests/unit/utils/test_http.py
@@ -16,7 +16,7 @@ from tests.support.runtests import RUNTIME_VARS
 from tests.support.unit import TestCase
 
 try:
-    import salt.ext.tornado.curl_httpclient  # pylint: disable=unused-import
+    import tornado.curl_httpclient  # pylint: disable=unused-import
 
     HAS_CURL = True
 except ImportError:

From 2e1097eac622a32346444defc112d568d961b805 Mon Sep 17 00:00:00 2001
From: "Daniel A. Wozniak" 
Date: Thu, 18 May 2023 15:18:13 -0700
Subject: [PATCH 088/152] Fix cruft caught in review

---
 salt/netapi/rest_tornado/saltnado_websockets.py | 2 +-
 salt/version.py                                 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py
index 1e887a8f64c..32ae448ff6e 100644
--- a/salt/netapi/rest_tornado/saltnado_websockets.py
+++ b/salt/netapi/rest_tornado/saltnado_websockets.py
@@ -395,7 +395,7 @@ class FormattedEventsHandler(AllEventsHandler):  # pylint: disable=W0223,W0232
         These messages make up salt's
         "real time" event stream.
         """
-        log.error("Got websocket message %s", message)
+        log.debug("Got websocket message %s", message)
         if message == "websocket client ready":
             if self.connected:
                 # TBD: Add ability to run commands in this branch
diff --git a/salt/version.py b/salt/version.py
index 1962510b6da..43cb5f86f75 100644
--- a/salt/version.py
+++ b/salt/version.py
@@ -79,7 +79,7 @@ class SaltVersionsInfo(type):
     SILICON       = SaltVersion("Silicon"      , info=3004,       released=True)
     PHOSPHORUS    = SaltVersion("Phosphorus"   , info=3005,       released=True)
     SULFUR        = SaltVersion("Sulfur"       , info=(3006, 0),  released=True)
-    CHLORINE      = SaltVersion("Chlorine"     , info=(3007, 0),  released=True)
+    CHLORINE      = SaltVersion("Chlorine"     , info=(3007, 0))
     ARGON         = SaltVersion("Argon"        , info=(3008, 0))
     POTASSIUM     = SaltVersion("Potassium"    , info=(3009, 0))
     CALCIUM       = SaltVersion("Calcium"      , info=(3010, 0))

From 117cd6861dc88bfccc867aa6301241a4790f9e5f Mon Sep 17 00:00:00 2001
From: "Daniel A. Wozniak" 
Date: Thu, 18 May 2023 15:18:26 -0700
Subject: [PATCH 089/152] Add changelog for tornado upgrade

---
 changelog/64305.fixed.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog/64305.fixed.md

diff --git a/changelog/64305.fixed.md b/changelog/64305.fixed.md
new file mode 100644
index 00000000000..dafe2e91b56
--- /dev/null
+++ b/changelog/64305.fixed.md
@@ -0,0 +1 @@
+Upgade to a recent tornado version 6.1

From 1f145c0f6e8f82aff9cacb5c04db3e5f11bb73cc Mon Sep 17 00:00:00 2001
From: "Daniel A. Wozniak" 
Date: Thu, 18 May 2023 20:10:52 -0700
Subject: [PATCH 090/152] Do not fail when cleaning up loop file handlers

---
 salt/utils/asynchronous.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/salt/utils/asynchronous.py b/salt/utils/asynchronous.py
index e887cb693f0..0b7ac540a92 100644
--- a/salt/utils/asynchronous.py
+++ b/salt/utils/asynchronous.py
@@ -108,7 +108,10 @@ class SyncWrapper:
                 log.exception("Exception encountered while running stop method")
         io_loop = self.io_loop
         io_loop.stop()
-        io_loop.close(all_fds=True)
+        try:
+            io_loop.close(all_fds=True)
+        except KeyError:
+            pass
 
     def __getattr__(self, key):
         if key in self._async_methods:

From 5ad05c61cf7d001e42d7a53bfeb42496e5d384ff Mon Sep 17 00:00:00 2001
From: "Daniel A. Wozniak" 
Date: Fri, 19 May 2023 00:29:54 -0700
Subject: [PATCH 091/152] Fix up based on PR reviews

---
 salt/metaproxy/deltaproxy.py | 6 +++---
 salt/metaproxy/proxy.py      | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/salt/metaproxy/deltaproxy.py b/salt/metaproxy/deltaproxy.py
index a9a7fa50a18..223b611e959 100644
--- a/salt/metaproxy/deltaproxy.py
+++ b/salt/metaproxy/deltaproxy.py
@@ -11,8 +11,8 @@ import threading
 import traceback
 import types
 
-import tornado.gen  # pylint: disable=F0401
-import tornado.ioloop  # pylint: disable=F0401
+import tornado.gen
+import tornado.ioloop
 
 import salt
 import salt._logging
@@ -355,7 +355,7 @@ def post_master_init(self, master):
         try:
             results = yield tornado.gen.multi(waitfor)
         except Exception as exc:  # pylint: disable=broad-except
-            log.error("Errors loading sub proxies")
+            log.error("Errors loading sub proxies: %s", exc)
 
         _failed = self.opts["proxy"].get("ids", [])[:]
         for sub_proxy_data in results:
diff --git a/salt/metaproxy/proxy.py b/salt/metaproxy/proxy.py
index ace2260cf62..c83fb8f959c 100644
--- a/salt/metaproxy/proxy.py
+++ b/salt/metaproxy/proxy.py
@@ -9,8 +9,8 @@ import threading
 import traceback
 import types
 
-import tornado.gen  # pylint: disable=F0401
-import tornado.ioloop  # pylint: disable=F0401
+import tornado.gen
+import tornado.ioloop
 
 import salt
 import salt.beacons

From 356120f91aac3f902ac45cb4a4f4764835771207 Mon Sep 17 00:00:00 2001
From: "Daniel A. Wozniak" 
Date: Mon, 22 May 2023 14:05:36 -0700
Subject: [PATCH 092/152] Revert unwanted changes dependent on other PRs

---
 doc/topics/releases/index.rst | 3 ++-
 tools/vm.py                   | 2 --
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/topics/releases/index.rst b/doc/topics/releases/index.rst
index 725f578d2f7..cf1981611d6 100644
--- a/doc/topics/releases/index.rst
+++ b/doc/topics/releases/index.rst
@@ -19,7 +19,7 @@ Upcoming release
     :maxdepth: 1
     :glob:
 
-    3006.*
+    3007.*
 
 See `Install a release candidate `_
 for more information about installing an RC when one is available.
@@ -31,6 +31,7 @@ Previous releases
     :maxdepth: 1
     :glob:
 
+    3006.*
     3005*
     3004*
     3003*
diff --git a/tools/vm.py b/tools/vm.py
index 70875ad42d4..f7b2837ae1b 100644
--- a/tools/vm.py
+++ b/tools/vm.py
@@ -450,8 +450,6 @@ def install_dependencies(ctx: Context, name: str, nox_session: str = "ci-test-3"
     Install test dependencies on VM.
     """
     vm = VM(ctx=ctx, name=name, region_name=ctx.parser.options.region)
-    if name == "amazonlinux-2":
-        vm.run(["sudo", "yum", "install", "-y", "libffi-devel"])
     returncode = vm.install_dependencies(nox_session)
     ctx.exit(returncode)
 

From 23582dce207e1c4623ec8c331ee79ea353ad2a25 Mon Sep 17 00:00:00 2001
From: "Daniel A. Wozniak" 
Date: Mon, 22 May 2023 14:56:15 -0700
Subject: [PATCH 093/152] The linter is not always right

---
 tests/pytests/unit/states/test_pip.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py
index 307ba5e1e65..23ac8436164 100644
--- a/tests/pytests/unit/states/test_pip.py
+++ b/tests/pytests/unit/states/test_pip.py
@@ -67,5 +67,8 @@ def test_issue_64169(caplog):
 
         # Confirm that the state continued to install the package as expected.
         # Only check the 'pkgs' parameter of pip.install
-        mock_install_call_args, mock_install_call_kwargs = mock_pip_install.call_args
+        (  # pylint: disable=unpacking-non-sequence
+            mock_install_call_args,
+            mock_install_call_kwargs,
+        ) = mock_pip_install.call_args
         assert mock_install_call_kwargs["pkgs"] == pkg_to_install

From 85b51e444964290e809abea6f2f383304eb2f99f Mon Sep 17 00:00:00 2001
From: cmcmarrow 
Date: Tue, 23 May 2023 11:22:14 -0500
Subject: [PATCH 094/152] fix 64169 lint error

---
 tests/pytests/unit/states/test_pip.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py
index 307ba5e1e65..1a71be86ac1 100644
--- a/tests/pytests/unit/states/test_pip.py
+++ b/tests/pytests/unit/states/test_pip.py
@@ -67,5 +67,4 @@ def test_issue_64169(caplog):
 
         # Confirm that the state continued to install the package as expected.
         # Only check the 'pkgs' parameter of pip.install
-        mock_install_call_args, mock_install_call_kwargs = mock_pip_install.call_args
-        assert mock_install_call_kwargs["pkgs"] == pkg_to_install
+        assert mock_pip_install.call_args.kwargs["pkgs"] == pkg_to_install

From 7f64ec3db2ba08da9c7017b6eb945a6f8b951f73 Mon Sep 17 00:00:00 2001
From: Pedro Algarvio 
Date: Wed, 24 May 2023 06:59:55 +0100
Subject: [PATCH 095/152] Add changelog for PR #64315

Signed-off-by: Pedro Algarvio 
---
 changelog/64315.removed.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog/64315.removed.md

diff --git a/changelog/64315.removed.md b/changelog/64315.removed.md
new file mode 100644
index 00000000000..3c00d4c2c27
--- /dev/null
+++ b/changelog/64315.removed.md
@@ -0,0 +1 @@
+Fedora 36 support was removed because it reached EOL

From 9a15e2285eb9f1a6ca9d17e6ea9fee1878c2b6d1 Mon Sep 17 00:00:00 2001
From: MKLeb 
Date: Thu, 20 Apr 2023 12:02:19 -0400
Subject: [PATCH 096/152] Change the matrix values for better actions reading
 when building from an existing onedir or not

---
 .github/workflows/build-deb-packages.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-deb-packages.yml b/.github/workflows/build-deb-packages.yml
index 3823a620ed1..1de426ffc40 100644
--- a/.github/workflows/build-deb-packages.yml
+++ b/.github/workflows/build-deb-packages.yml
@@ -77,7 +77,7 @@ jobs:
         run: |
           tools pkg apply-release-patch salt-${{ inputs.salt-version }}.patch --delete
 
-      - name: Build Deb
+      - name: Build Deb Without Existing Onedir
         working-directory: pkgs/checkout/
         run: |
           tools pkg build deb ${{

From c6c9b7fefc77f618ed72cda110bc7a85b7d7e3b6 Mon Sep 17 00:00:00 2001
From: MKLeb 
Date: Thu, 20 Apr 2023 13:02:08 -0400
Subject: [PATCH 097/152] Build windows packages without an existing onedir in
 CICD

---
 .github/workflows/build-windows-packages.yml | 33 ++++++++++++-
 tools/pkg/build.py                           | 50 +++++++++++---------
 2 files changed, 60 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/build-windows-packages.yml b/.github/workflows/build-windows-packages.yml
index b50d7cdc618..c0ebe9a62ac 100644
--- a/.github/workflows/build-windows-packages.yml
+++ b/.github/workflows/build-windows-packages.yml
@@ -29,6 +29,10 @@ jobs:
         arch:
           - x86
           - amd64
+        mode:
+          - existing-onedir
+          - from-src
+
     runs-on:
       - windows-latest
     env:
@@ -93,25 +97,52 @@ jobs:
         run: |
           echo "${{ secrets.WIN_SIGN_CERT_FILE_B64 }}" | base64 --decode > /d/Certificate_pkcs12.p12
 
-      - name: Build Windows Packages
+      - name: Build Windows Packages Using Existing Onedir
+        if: ${{ matrix.mode == 'existing-onedir' }}
         run: |
           tools pkg build windows --onedir salt-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.zip `
             --salt-version ${{ inputs.salt-version }} --arch ${{ matrix.arch }} ${{
               steps.check-pkg-sign.outputs.sign-pkgs == 'true' && '--sign' || ''
             }}
 
+      - name: Build Windows Packages Without Existing Onedir
+        if: ${{ matrix.mode == 'from-src' }}
+        run: |
+          tools pkg build windows --onedir salt-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.zip `
+            --salt-version ${{ inputs.salt-version }} --arch ${{ matrix.arch }}
+
       - name: Upload ${{ matrix.arch }} Packages
         uses: actions/upload-artifact@v3
+        if: ${{ matrix.mode == 'existing-onedir' }}
         with:
           name: salt-${{ inputs.salt-version }}-${{ matrix.arch }}-NSIS
           path: pkg/windows/build/Salt-*.exe
           retention-days: 7
           if-no-files-found: error
 
+      - name: Upload ${{ matrix.arch }} Packages
+        uses: actions/upload-artifact@v3
+        if: ${{ matrix.mode == 'from-src' }}
+        with:
+          name: salt-${{ inputs.salt-version }}-${{ matrix.arch }}-NSIS-from-src
+          path: pkg/windows/build/Salt-*.exe
+          retention-days: 7
+          if-no-files-found: error
+
       - name: Upload ${{ matrix.arch }} MSI Package
         uses: actions/upload-artifact@v3
+        if: ${{ matrix.mode == 'existing-onedir' }}
         with:
           name: salt-${{ inputs.salt-version }}-${{ matrix.arch }}-MSI
           path: pkg/windows/build/Salt-*.msi
           retention-days: 7
           if-no-files-found: error
+
+      - name: Upload ${{ matrix.arch }} MSI Package
+        uses: actions/upload-artifact@v3
+        if: ${{ matrix.mode == 'from-src' }}
+        with:
+          name: salt-${{ inputs.salt-version }}-${{ matrix.arch }}-MSI-from-src
+          path: pkg/windows/build/Salt-*.msi
+          retention-days: 7
+          if-no-files-found: error
diff --git a/tools/pkg/build.py b/tools/pkg/build.py
index b373338a99e..edc2e2d73a9 100644
--- a/tools/pkg/build.py
+++ b/tools/pkg/build.py
@@ -222,7 +222,6 @@ def macos(
     arguments={
         "onedir": {
             "help": "The name of the onedir artifact, if given it should be under artifacts/",
-            "required": True,
         },
         "salt_version": {
             "help": (
@@ -237,7 +236,7 @@ def macos(
             "required": True,
         },
         "sign": {
-            "help": "Sign and notorize built package",
+            "help": "Sign and notarize built package",
         },
     },
 )
@@ -252,28 +251,10 @@ def windows(
     Build the Windows package.
     """
     if TYPE_CHECKING:
-        assert onedir is not None
         assert salt_version is not None
         assert arch is not None
 
-    checkout = pathlib.Path.cwd()
-    onedir_artifact = checkout / "artifacts" / onedir
-    _check_pkg_build_files_exist(ctx, onedir_artifact=onedir_artifact)
-
-    unzip_dir = checkout / "pkg" / "windows"
-    ctx.info(f"Unzipping the onedir artifact to {unzip_dir}")
-    with zipfile.ZipFile(onedir_artifact, mode="r") as archive:
-        archive.extractall(unzip_dir)
-
-    move_dir = unzip_dir / "salt"
-    build_env = unzip_dir / "buildenv"
-    _check_pkg_build_files_exist(ctx, move_dir=move_dir)
-
-    ctx.info(f"Moving {move_dir} directory to the build environment in {build_env}")
-    shutil.move(move_dir, build_env)
-
-    ctx.info("Building the windows package")
-    ctx.run(
+    build_cmd = [
         "powershell.exe",
         "&",
         "pkg/windows/build.cmd",
@@ -283,7 +264,32 @@ def windows(
         salt_version,
         "-CICD",
         "-SkipInstall",
-    )
+    ]
+
+    if onedir:
+        build_cmd.append("-SkipInstall")
+        checkout = pathlib.Path.cwd()
+        onedir_artifact = checkout / "artifacts" / onedir
+        ctx.info(f"Building package from existing onedir: {str(onedir_artifact)}")
+        _check_pkg_build_files_exist(ctx, onedir_artifact=onedir_artifact)
+
+        unzip_dir = checkout / "pkg" / "windows"
+        ctx.info(f"Unzipping the onedir artifact to {unzip_dir}")
+        with zipfile.ZipFile(onedir_artifact, mode="r") as archive:
+            archive.extractall(unzip_dir)
+
+        move_dir = unzip_dir / "salt"
+        build_env = unzip_dir / "buildenv"
+        _check_pkg_build_files_exist(ctx, move_dir=move_dir)
+
+        ctx.info(f"Moving {move_dir} directory to the build environment in {build_env}")
+        shutil.move(move_dir, build_env)
+    else:
+        build_cmd.append("-Build")
+        ctx.info("Building package without an existing onedir")
+
+    ctx.info(f"Running: {' '.join(build_cmd)} ...")
+    ctx.run(*build_cmd)
 
     if sign:
         env = os.environ.copy()

From e2a624f9845aad3341573e05c1e37680274a6060 Mon Sep 17 00:00:00 2001
From: MKLeb 
Date: Thu, 20 Apr 2023 14:55:31 -0400
Subject: [PATCH 098/152] Build macos packages without existing onedir

---
 .github/workflows/build-macos-packages.yml   | 22 ++++++++++-
 .github/workflows/build-windows-packages.yml |  3 +-
 pkg/macos/build_python.sh                    | 41 +++-----------------
 tools/pkg/build.py                           | 35 +++++++++++------
 4 files changed, 51 insertions(+), 50 deletions(-)

diff --git a/.github/workflows/build-macos-packages.yml b/.github/workflows/build-macos-packages.yml
index 9e07834fea0..39bd3714c06 100644
--- a/.github/workflows/build-macos-packages.yml
+++ b/.github/workflows/build-macos-packages.yml
@@ -27,6 +27,10 @@ jobs:
       matrix:
         arch:
           - x86_64
+        mode:
+          - existing-onedir
+          - from-src
+
     runs-on:
       - macos-12
     steps:
@@ -93,22 +97,38 @@ jobs:
           rm install-cert.p12
           security set-key-partition-list -S apple-tool:,apple: -k "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" &> /dev/null
 
-      - name: Build MacOS Package
+      - name: Build MacOS Package Using Existing Onedir
         env:
           DEV_APP_CERT: "${{ secrets.MAC_SIGN_DEV_APP_CERT }}"
           DEV_INSTALL_CERT: "${{ secrets.MAC_SIGN_DEV_INSTALL_CERT }}"
           APPLE_ACCT: "${{ secrets.MAC_SIGN_APPLE_ACCT }}"
           APP_SPEC_PWD: "${{ secrets.MAC_SIGN_APP_SPEC_PWD }}"
+        if: ${{ matrix.mode == 'existing-onedir' }}
         run: |
           tools pkg build macos --onedir salt-${{ inputs.salt-version }}-onedir-darwin-${{ matrix.arch }}.tar.xz \
             --salt-version ${{ inputs.salt-version }} ${{
               steps.check-pkg-sign.outputs.sign-pkgs == 'true' && '--sign' || ''
             }}
 
+      - name: Build MacOS Package Without Existing Onedir
+        if: ${{ matrix.mode == 'from-src' }}
+        run: |
+          tools pkg build macos --salt-version ${{ inputs.salt-version }}
+
       - name: Upload ${{ matrix.arch }} Package
         uses: actions/upload-artifact@v3
+        if: ${{ matrix.mode == 'existing-onedir' }}
         with:
           name: salt-${{ inputs.salt-version }}-${{ matrix.arch }}-macos
           path: pkg/macos/salt-${{ inputs.salt-version }}-py3-*.pkg
           retention-days: 7
           if-no-files-found: error
+
+      - name: Upload ${{ matrix.arch }} Package
+        uses: actions/upload-artifact@v3
+        if: ${{ matrix.mode == 'from-src' }}
+        with:
+          name: salt-${{ inputs.salt-version }}-${{ matrix.arch }}-macos-from-src
+          path: pkg/macos/salt-${{ inputs.salt-version }}-py3-*.pkg
+          retention-days: 7
+          if-no-files-found: error
diff --git a/.github/workflows/build-windows-packages.yml b/.github/workflows/build-windows-packages.yml
index c0ebe9a62ac..608c17d7d77 100644
--- a/.github/workflows/build-windows-packages.yml
+++ b/.github/workflows/build-windows-packages.yml
@@ -108,8 +108,7 @@ jobs:
       - name: Build Windows Packages Without Existing Onedir
         if: ${{ matrix.mode == 'from-src' }}
         run: |
-          tools pkg build windows --onedir salt-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.zip `
-            --salt-version ${{ inputs.salt-version }} --arch ${{ matrix.arch }}
+          tools pkg build windows --salt-version ${{ inputs.salt-version }} --arch ${{ matrix.arch }}
 
       - name: Upload ${{ matrix.arch }} Packages
         uses: actions/upload-artifact@v3
diff --git a/pkg/macos/build_python.sh b/pkg/macos/build_python.sh
index 23fce00eabb..4be9a0f3705 100755
--- a/pkg/macos/build_python.sh
+++ b/pkg/macos/build_python.sh
@@ -21,26 +21,6 @@
 # The default version to be built
 # TODO: The is not selectable via RELENV yet. This has to match whatever relenv
 # TODO: is building
-PY_VERSION="3.10.9"
-
-# Valid versions supported by macOS
-PY_VERSIONS=(
-    "3.10.9"
-    "3.10.8"
-    "3.10.7"
-    "3.9.16"
-    "3.9.15"
-    "3.9.14"
-    "3.9.13"
-    "3.9.12"
-    "3.9.11"
-    "3.8.16"
-    "3.8.15"
-    "3.8.14"
-    "3.8.13"
-    "3.8.12"
-    "3.8.11"
-)
 
 # Locations
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -65,14 +45,10 @@ _usage() {
      echo ""
      echo "  -h, --help      this message"
      echo "  -b, --build     build python instead of fetching"
-     echo "  -v, --version   version of python to install"
-     echo "                  python version must be one of:"
-     for i in "${PY_VERSIONS[@]}"; do
-         echo "                  - $i"
-     done
+     echo "  -v, --version   version of python to install, must be available with relenv"
      echo ""
-     echo "  To build python 3.9.15:"
-     echo "      example: $0 --version 3.9.15"
+     echo "  To build python 3.10.11:"
+     echo "      example: $0 --version 3.10.11"
 }
 
 # _msg
@@ -129,13 +105,6 @@ while true; do
     esac
 done
 
-if ! [[ " ${PY_VERSIONS[*]} " =~ " $PY_VERSION " ]]; then
-    echo "Invalid Python Version: $PY_VERSION"
-    echo ""
-    _usage
-    exit 1
-fi
-
 #-------------------------------------------------------------------------------
 # Script Start
 #-------------------------------------------------------------------------------
@@ -231,8 +200,8 @@ else
     # We want to suppress the output here so it looks nice
     # To see the output, remove the output redirection
     _msg "Fetching python (relenv)"
-    relenv fetch >/dev/null 2>&1
-    if [ -f "$RELENV_DIR/build/x86_64-macos.tar.xz" ]; then
+    relenv fetch --python $PY_VERSION >/dev/null 2>&1
+    if [ -f "$RELENV_DIR/build/$PY_VERSION-x86_64-macos.tar.xz" ]; then
         _success
     else
         _failure
diff --git a/tools/pkg/build.py b/tools/pkg/build.py
index edc2e2d73a9..00df5bb579b 100644
--- a/tools/pkg/build.py
+++ b/tools/pkg/build.py
@@ -160,7 +160,6 @@ def rpm(
     arguments={
         "onedir": {
             "help": "The name of the onedir artifact, if given it should be under artifacts/",
-            "required": True,
         },
         "salt_version": {
             "help": (
@@ -184,16 +183,31 @@ def macos(
         assert onedir is not None
         assert salt_version is not None
 
-    checkout = pathlib.Path.cwd()
-    onedir_artifact = checkout / "artifacts" / onedir
-    _check_pkg_build_files_exist(ctx, onedir_artifact=onedir_artifact)
+    if onedir:
+        checkout = pathlib.Path.cwd()
+        onedir_artifact = checkout / "artifacts" / onedir
+        ctx.info(f"Building package from existing onedir: {str(onedir_artifact)}")
+        _check_pkg_build_files_exist(ctx, onedir_artifact=onedir_artifact)
 
-    build_root = checkout / "pkg" / "macos" / "build" / "opt"
-    build_root.mkdir(parents=True, exist_ok=True)
-    ctx.info(f"Extracting the onedir artifact to {build_root}")
-    with tarfile.open(str(onedir_artifact)) as tarball:
-        with ctx.chdir(onedir_artifact.parent):
-            tarball.extractall(path=build_root)
+        build_root = checkout / "pkg" / "macos" / "build" / "opt"
+        build_root.mkdir(parents=True, exist_ok=True)
+        ctx.info(f"Extracting the onedir artifact to {build_root}")
+        with tarfile.open(str(onedir_artifact)) as tarball:
+            with ctx.chdir(onedir_artifact.parent):
+                tarball.extractall(path=build_root)
+    else:
+        ctx.info("Building package without an existing onedir")
+
+    if not onedir:
+        # Prep the salt onedir if not building from an existing one
+        shared_constants = _get_shared_constants()
+        py_ver = shared_constants["python_version_macos"]
+        with ctx.chdir(checkout / "pkg" / "macos"):
+            ctx.info("Fetching relenv python")
+            ctx.run("./build_python.sh", "--version", py_ver)
+
+            ctx.info("Installing salt into the relenv python")
+            ctx.run("./install_salt.sh")
 
     if sign:
         ctx.info("Signing binaries")
@@ -263,7 +277,6 @@ def windows(
         "-Version",
         salt_version,
         "-CICD",
-        "-SkipInstall",
     ]
 
     if onedir:

From ae9f9d379bb9a115929036ea18b198560875044f Mon Sep 17 00:00:00 2001
From: MKLeb 
Date: Tue, 25 Apr 2023 17:30:38 -0400
Subject: [PATCH 099/152] Fix windows and mac building from without an existing
 onedir

---
 pkg/windows/build.ps1 | 2 +-
 tools/pkg/build.py    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/windows/build.ps1 b/pkg/windows/build.ps1
index 0d87d604007..d9e7222d50d 100644
--- a/pkg/windows/build.ps1
+++ b/pkg/windows/build.ps1
@@ -167,7 +167,7 @@ if ( ! $SkipInstall ) {
       Architecture = $Architecture
   }
   if ( $Build ) {
-      $KeywordArguments["Build"] = $true
+      $KeywordArguments["Build"] = $false
   }
   if ( $CICD ) {
       $KeywordArguments["CICD"] = $true
diff --git a/tools/pkg/build.py b/tools/pkg/build.py
index 00df5bb579b..d4f2ded8dd2 100644
--- a/tools/pkg/build.py
+++ b/tools/pkg/build.py
@@ -183,8 +183,8 @@ def macos(
         assert onedir is not None
         assert salt_version is not None
 
+    checkout = pathlib.Path.cwd()
     if onedir:
-        checkout = pathlib.Path.cwd()
         onedir_artifact = checkout / "artifacts" / onedir
         ctx.info(f"Building package from existing onedir: {str(onedir_artifact)}")
         _check_pkg_build_files_exist(ctx, onedir_artifact=onedir_artifact)
@@ -279,9 +279,9 @@ def windows(
         "-CICD",
     ]
 
+    checkout = pathlib.Path.cwd()
     if onedir:
         build_cmd.append("-SkipInstall")
-        checkout = pathlib.Path.cwd()
         onedir_artifact = checkout / "artifacts" / onedir
         ctx.info(f"Building package from existing onedir: {str(onedir_artifact)}")
         _check_pkg_build_files_exist(ctx, onedir_artifact=onedir_artifact)

From 8fce8ca476f5c1235797232130bae5889c856203 Mon Sep 17 00:00:00 2001
From: MKLeb 
Date: Wed, 26 Apr 2023 09:18:10 -0400
Subject: [PATCH 100/152] Relenv now produces Python `3.10.11`

---
 pkg/windows/build.ps1        | 4 ++--
 pkg/windows/build_python.ps1 | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/windows/build.ps1 b/pkg/windows/build.ps1
index d9e7222d50d..9a4070a3bf4 100644
--- a/pkg/windows/build.ps1
+++ b/pkg/windows/build.ps1
@@ -40,7 +40,7 @@ param(
     [ValidatePattern("^\d{1,2}.\d{1,2}.\d{1,2}$")]
     [ValidateSet(
         "3.11.2",
-        "3.10.10"
+        "3.10.11"
     )]
     [Alias("p")]
     # The version of Python to be built. Pythonnet only supports up to Python
@@ -48,7 +48,7 @@ param(
     # supported up to 3.8. So we're pinned to the latest version of Python 3.8.
     # We may have to drop support for pycurl.
     # Default is: 3.8.16
-    [String] $PythonVersion = "3.10.10",
+    [String] $PythonVersion = "3.10.11",
 
     [Parameter(Mandatory=$false)]
     [Alias("b")]
diff --git a/pkg/windows/build_python.ps1 b/pkg/windows/build_python.ps1
index 35cdf1fb0e3..bb24425ba6b 100644
--- a/pkg/windows/build_python.ps1
+++ b/pkg/windows/build_python.ps1
@@ -19,7 +19,7 @@ param(
     [ValidatePattern("^\d{1,2}.\d{1,2}.\d{1,2}$")]
     [ValidateSet(
         "3.11.2",
-        "3.10.10"
+        "3.10.11"
     )]
     [Alias("v")]
     # The version of Python to be built. Pythonnet only supports up to Python
@@ -27,7 +27,7 @@ param(
     # supported up to 3.8. So we're pinned to the latest version of Python 3.8.
     # We may have to drop support for pycurl or build it ourselves.
     # Default is: 3.8.16
-    [String] $Version = "3.10.10",
+    [String] $Version = "3.10.11",
 
     [Parameter(Mandatory=$false)]
     [ValidateSet("x64", "x86", "amd64")]

From 42c5e3f6cbbfdf7c595dff3d14ddf5e0c904cb5a Mon Sep 17 00:00:00 2001
From: MKLeb 
Date: Mon, 1 May 2023 16:33:00 -0400
Subject: [PATCH 101/152] Fix title for deb build CI step

---
 .github/workflows/build-deb-packages.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-deb-packages.yml b/.github/workflows/build-deb-packages.yml
index 1de426ffc40..3823a620ed1 100644
--- a/.github/workflows/build-deb-packages.yml
+++ b/.github/workflows/build-deb-packages.yml
@@ -77,7 +77,7 @@ jobs:
         run: |
           tools pkg apply-release-patch salt-${{ inputs.salt-version }}.patch --delete
 
-      - name: Build Deb Without Existing Onedir
+      - name: Build Deb
         working-directory: pkgs/checkout/
         run: |
           tools pkg build deb ${{

From 2fcdde4ab99a36e1a4cd0d22fafe8c8f62fce9e7 Mon Sep 17 00:00:00 2001
From: MKLeb 
Date: Mon, 1 May 2023 16:35:11 -0400
Subject: [PATCH 102/152] Make the windows package builds follow the same
 format as the deb and rpm builds

---
 .github/workflows/build-windows-packages.yml | 57 +++++++++-----------
 1 file changed, 24 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/build-windows-packages.yml b/.github/workflows/build-windows-packages.yml
index 608c17d7d77..17ab26cde2c 100644
--- a/.github/workflows/build-windows-packages.yml
+++ b/.github/workflows/build-windows-packages.yml
@@ -29,9 +29,9 @@ jobs:
         arch:
           - x86
           - amd64
-        mode:
-          - existing-onedir
-          - from-src
+        source:
+          - onedir
+          - src
 
     runs-on:
       - windows-latest
@@ -98,50 +98,41 @@ jobs:
           echo "${{ secrets.WIN_SIGN_CERT_FILE_B64 }}" | base64 --decode > /d/Certificate_pkcs12.p12
 
       - name: Build Windows Packages Using Existing Onedir
-        if: ${{ matrix.mode == 'existing-onedir' }}
         run: |
-          tools pkg build windows --onedir salt-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.zip `
-            --salt-version ${{ inputs.salt-version }} --arch ${{ matrix.arch }} ${{
+          tools pkg build windows ${{
+            matrix.source == 'onedir' &&
+            format(
+              '--onedir=salt-{0}-onedir-linux-{1}.tar.xz --salt-version {0} --arch {1} {2}',
+              inputs.salt-version,
+              matrix.arch,
               steps.check-pkg-sign.outputs.sign-pkgs == 'true' && '--sign' || ''
-            }}
+            ) ||
+            format('--salt-version {0} --arch {1}', inputs.salt-version, matrix.arch)
+          }}
 
-      - name: Build Windows Packages Without Existing Onedir
-        if: ${{ matrix.mode == 'from-src' }}
+      - name: Set Artifact Name
+        id: set-artifact-name
         run: |
-          tools pkg build windows --salt-version ${{ inputs.salt-version }} --arch ${{ matrix.arch }}
+          if [ "${{ matrix.source }}" != "src" ]; then
+            echo "artifact-name-nsis=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-NSIS" >> "$GITHUB_OUTPUT"
+            echo "artifact-name-msi=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-MSI" >> "$GITHUB_OUTPUT"
+          else
+            echo "artifact-name-nsis=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-NSIS-from-src" >> "$GITHUB_OUTPUT"
+            echo "artifact-name-msi=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-MSI"-from-src >> "$GITHUB_OUTPUT"
+          fi
 
-      - name: Upload ${{ matrix.arch }} Packages
+      - name: Upload ${{ matrix.arch }} NSIS Packages
         uses: actions/upload-artifact@v3
-        if: ${{ matrix.mode == 'existing-onedir' }}
         with:
-          name: salt-${{ inputs.salt-version }}-${{ matrix.arch }}-NSIS
-          path: pkg/windows/build/Salt-*.exe
-          retention-days: 7
-          if-no-files-found: error
-
-      - name: Upload ${{ matrix.arch }} Packages
-        uses: actions/upload-artifact@v3
-        if: ${{ matrix.mode == 'from-src' }}
-        with:
-          name: salt-${{ inputs.salt-version }}-${{ matrix.arch }}-NSIS-from-src
+          name: ${{ steps.set-artifact-name.outputs.artifact-name-nsis }}
           path: pkg/windows/build/Salt-*.exe
           retention-days: 7
           if-no-files-found: error
 
       - name: Upload ${{ matrix.arch }} MSI Package
         uses: actions/upload-artifact@v3
-        if: ${{ matrix.mode == 'existing-onedir' }}
         with:
-          name: salt-${{ inputs.salt-version }}-${{ matrix.arch }}-MSI
-          path: pkg/windows/build/Salt-*.msi
-          retention-days: 7
-          if-no-files-found: error
-
-      - name: Upload ${{ matrix.arch }} MSI Package
-        uses: actions/upload-artifact@v3
-        if: ${{ matrix.mode == 'from-src' }}
-        with:
-          name: salt-${{ inputs.salt-version }}-${{ matrix.arch }}-MSI-from-src
+          name: ${{ steps.set-artifact-name.outputs.artifact-name-msi }}
           path: pkg/windows/build/Salt-*.msi
           retention-days: 7
           if-no-files-found: error

From 2c6b0fbe4a64f5aaf33c1f74f7e6e2b42cbb0eb4 Mon Sep 17 00:00:00 2001
From: MKLeb 
Date: Mon, 1 May 2023 16:41:51 -0400
Subject: [PATCH 103/152] Make the macos package builds follow the same format
 as the deb and rpm builds

---
 .github/workflows/build-macos-packages.yml   | 44 ++++++++++----------
 .github/workflows/build-windows-packages.yml |  5 ++-
 2 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/build-macos-packages.yml b/.github/workflows/build-macos-packages.yml
index 39bd3714c06..ab7a42a0f4b 100644
--- a/.github/workflows/build-macos-packages.yml
+++ b/.github/workflows/build-macos-packages.yml
@@ -27,9 +27,9 @@ jobs:
       matrix:
         arch:
           - x86_64
-        mode:
-          - existing-onedir
-          - from-src
+        source:
+          - onedir
+          - src
 
     runs-on:
       - macos-12
@@ -103,32 +103,32 @@ jobs:
           DEV_INSTALL_CERT: "${{ secrets.MAC_SIGN_DEV_INSTALL_CERT }}"
           APPLE_ACCT: "${{ secrets.MAC_SIGN_APPLE_ACCT }}"
           APP_SPEC_PWD: "${{ secrets.MAC_SIGN_APP_SPEC_PWD }}"
-        if: ${{ matrix.mode == 'existing-onedir' }}
         run: |
-          tools pkg build macos --onedir salt-${{ inputs.salt-version }}-onedir-darwin-${{ matrix.arch }}.tar.xz \
-            --salt-version ${{ inputs.salt-version }} ${{
-              steps.check-pkg-sign.outputs.sign-pkgs == 'true' && '--sign' || ''
-            }}
+          tools pkg build macos ${{
+              matrix.source == 'onedir' &&
+              format(
+                '--onedir salt-{0}-onedir-darwin-{1}.tar.xz --salt-version {0} {2}',
+                inputs.salt-version,
+                matrix.arch,
+                steps.check-pkg-sign.outputs.sign-pkgs == 'true' && '--sign' || ''
+              )
+              ||
+              format('--salt-version {0}', inputs.salt-version)
+          }}
 
-      - name: Build MacOS Package Without Existing Onedir
-        if: ${{ matrix.mode == 'from-src' }}
+      - name: Set Artifact Name
+        id: set-artifact-name
         run: |
-          tools pkg build macos --salt-version ${{ inputs.salt-version }}
+          if [ "${{ matrix.source }}" != "src" ]; then
+            echo "artifact-name=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-macos" >> "$GITHUB_OUTPUT"
+          else
+            echo "artifact-name=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-macos-from-src" >> "$GITHUB_OUTPUT"
+          fi
 
       - name: Upload ${{ matrix.arch }} Package
         uses: actions/upload-artifact@v3
-        if: ${{ matrix.mode == 'existing-onedir' }}
         with:
-          name: salt-${{ inputs.salt-version }}-${{ matrix.arch }}-macos
-          path: pkg/macos/salt-${{ inputs.salt-version }}-py3-*.pkg
-          retention-days: 7
-          if-no-files-found: error
-
-      - name: Upload ${{ matrix.arch }} Package
-        uses: actions/upload-artifact@v3
-        if: ${{ matrix.mode == 'from-src' }}
-        with:
-          name: salt-${{ inputs.salt-version }}-${{ matrix.arch }}-macos-from-src
+          name: ${{ steps.set-artifact-name.outputs.artifact-name }}
           path: pkg/macos/salt-${{ inputs.salt-version }}-py3-*.pkg
           retention-days: 7
           if-no-files-found: error
diff --git a/.github/workflows/build-windows-packages.yml b/.github/workflows/build-windows-packages.yml
index 17ab26cde2c..8aab6c1b173 100644
--- a/.github/workflows/build-windows-packages.yml
+++ b/.github/workflows/build-windows-packages.yml
@@ -106,7 +106,8 @@ jobs:
               inputs.salt-version,
               matrix.arch,
               steps.check-pkg-sign.outputs.sign-pkgs == 'true' && '--sign' || ''
-            ) ||
+            )
+            ||
             format('--salt-version {0} --arch {1}', inputs.salt-version, matrix.arch)
           }}
 
@@ -118,7 +119,7 @@ jobs:
             echo "artifact-name-msi=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-MSI" >> "$GITHUB_OUTPUT"
           else
             echo "artifact-name-nsis=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-NSIS-from-src" >> "$GITHUB_OUTPUT"
-            echo "artifact-name-msi=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-MSI"-from-src >> "$GITHUB_OUTPUT"
+            echo "artifact-name-msi=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-MSI-from-src" >> "$GITHUB_OUTPUT"
           fi
 
       - name: Upload ${{ matrix.arch }} NSIS Packages

From bd0b7171f7dca53f93f4b6f125f1ac5c4db3f589 Mon Sep 17 00:00:00 2001
From: MKLeb 
Date: Tue, 2 May 2023 12:05:09 -0400
Subject: [PATCH 104/152] Give the right onedir artifact name for windows
 package builds

---
 .github/workflows/build-windows-packages.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-windows-packages.yml b/.github/workflows/build-windows-packages.yml
index 8aab6c1b173..de110e38368 100644
--- a/.github/workflows/build-windows-packages.yml
+++ b/.github/workflows/build-windows-packages.yml
@@ -102,7 +102,7 @@ jobs:
           tools pkg build windows ${{
             matrix.source == 'onedir' &&
             format(
-              '--onedir=salt-{0}-onedir-linux-{1}.tar.xz --salt-version {0} --arch {1} {2}',
+              '--onedir salt-{0}-onedir-windows-{1}.zip --salt-version {0} --arch {1} {2}',
               inputs.salt-version,
               matrix.arch,
               steps.check-pkg-sign.outputs.sign-pkgs == 'true' && '--sign' || ''

From 724c498e94dae06d49cf93e58763e8c568a36e8a Mon Sep 17 00:00:00 2001
From: MKLeb 
Date: Tue, 2 May 2023 14:47:02 -0400
Subject: [PATCH 105/152] Set the artifact name for the windows packages using
 a bash shell

---
 .github/workflows/build-windows-packages.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-windows-packages.yml b/.github/workflows/build-windows-packages.yml
index de110e38368..f4f42009ebe 100644
--- a/.github/workflows/build-windows-packages.yml
+++ b/.github/workflows/build-windows-packages.yml
@@ -113,6 +113,7 @@ jobs:
 
       - name: Set Artifact Name
         id: set-artifact-name
+        shell: bash
         run: |
           if [ "${{ matrix.source }}" != "src" ]; then
             echo "artifact-name-nsis=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-NSIS" >> "$GITHUB_OUTPUT"

From d0130e6f767cb0d92894e27055ea8673e6a4d267 Mon Sep 17 00:00:00 2001
From: MKLeb 
Date: Tue, 2 May 2023 19:19:29 -0400
Subject: [PATCH 106/152] Add package building instructions for macos and
 windows

---
 .github/workflows/build-macos-packages.yml   |  2 +-
 .github/workflows/build-windows-packages.yml |  2 +-
 doc/topics/packaging/index.rst               | 78 ++++++++++++++++----
 3 files changed, 66 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/build-macos-packages.yml b/.github/workflows/build-macos-packages.yml
index ab7a42a0f4b..3b919abae5a 100644
--- a/.github/workflows/build-macos-packages.yml
+++ b/.github/workflows/build-macos-packages.yml
@@ -97,7 +97,7 @@ jobs:
           rm install-cert.p12
           security set-key-partition-list -S apple-tool:,apple: -k "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" &> /dev/null
 
-      - name: Build MacOS Package Using Existing Onedir
+      - name: Build MacOS Package
         env:
           DEV_APP_CERT: "${{ secrets.MAC_SIGN_DEV_APP_CERT }}"
           DEV_INSTALL_CERT: "${{ secrets.MAC_SIGN_DEV_INSTALL_CERT }}"
diff --git a/.github/workflows/build-windows-packages.yml b/.github/workflows/build-windows-packages.yml
index f4f42009ebe..34806090e9f 100644
--- a/.github/workflows/build-windows-packages.yml
+++ b/.github/workflows/build-windows-packages.yml
@@ -97,7 +97,7 @@ jobs:
         run: |
           echo "${{ secrets.WIN_SIGN_CERT_FILE_B64 }}" | base64 --decode > /d/Certificate_pkcs12.p12
 
-      - name: Build Windows Packages Using Existing Onedir
+      - name: Build Windows Packages
         run: |
           tools pkg build windows ${{
             matrix.source == 'onedir' &&
diff --git a/doc/topics/packaging/index.rst b/doc/topics/packaging/index.rst
index b16e062fa6c..58ac77b4f9f 100644
--- a/doc/topics/packaging/index.rst
+++ b/doc/topics/packaging/index.rst
@@ -13,7 +13,7 @@ are built with the `relenv `_.
 
 
@@ -53,6 +53,13 @@ How to build onedir only
 
 How to build rpm packages
 =========================
+
+#. Ensure you are in the current Salt cloned git repo:
+
+    .. code-block:: bash
+
+       cd 
+
 #. Install the dependencies:
 
     .. code-block:: bash
@@ -72,12 +79,6 @@ How to build rpm packages
 
        pip install -r requirements/static/ci/py{python_version}/changelog.txt
 
-#. Ensure you are in the current Salt cloned git repo:
-
-    .. code-block:: bash
-
-       cd 
-
 #. (Optional) To build a specific Salt version, run tools and set Salt version:
 
     .. code-block:: bash
@@ -93,10 +94,15 @@ How to build rpm packages
        tools pkg build rpm --relenv-version  --python-version  --arch 
 
 
-
 How to build deb packages
 =========================
 
+#. Ensure you are in the current Salt cloned git repo.:
+
+    .. code-block:: bash
+
+       cd 
+
 #. Install the dependencies:
 
     .. code-block:: bash
@@ -113,12 +119,6 @@ How to build deb packages
 
        pip install -r requirements/static/ci/py{python_version}/changelog.txt
 
-#. Ensure you are in the current Salt cloned git repo.:
-
-    .. code-block:: bash
-
-       cd 
-
 #. (Optional) To build a specific Salt version, run tools and set Salt version:
 
     .. code-block:: bash
@@ -135,6 +135,56 @@ How to build deb packages
        tools pkg build deb --relenv-version  --python-version  --arch 
 
 
+How to build MacOS packages
+===========================
+
+#. Ensure you are in the current Salt cloned git repo.:
+
+    .. code-block:: bash
+
+       cd 
+
+#. Install the dependencies:
+
+    .. code-block:: bash
+
+       pip install -r requirements/static/ci/py{python_version}/tools.txt
+
+#. Build the MacOS package:
+
+    Only the salt-version argument is required, the rest are optional.
+    Do note that you will not be able to sign the packages when building them.
+
+    .. code-block:: bash
+
+       tools pkg build macos --salt-version 
+
+
+How to build Windows packages
+=============================
+
+#. Ensure you are in the current Salt cloned git repo.:
+
+    .. code-block:: bash
+
+       cd 
+
+#. Install the dependencies:
+
+    .. code-block:: bash
+
+       pip install -r requirements/static/ci/py{python_version}/tools.txt
+
+#. Build the MacOS package:
+
+    Only the arch and salt-version arguments are required, the rest are optional.
+    Do note that you will not be able to sign the packages when building them.
+
+    .. code-block:: bash
+
+       tools pkg build windows --salt-version  --arch 
+
+
 How to access python binary
 ===========================
 

From 235cd933265d7ce7f1519bbdfc6990da3bb413be Mon Sep 17 00:00:00 2001
From: Natalie Lee 
Date: Thu, 25 May 2023 14:53:45 +0000
Subject: [PATCH 107/152] removing added pylint exception for
 unpacking-non-sequence pylint rule

---
 tests/pytests/unit/states/test_pip.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/pytests/unit/states/test_pip.py b/tests/pytests/unit/states/test_pip.py
index 4624ab25acb..44954033cb2 100644
--- a/tests/pytests/unit/states/test_pip.py
+++ b/tests/pytests/unit/states/test_pip.py
@@ -67,10 +67,9 @@ def test_issue_64169(caplog):
 
         # Confirm that the state continued to install the package as expected.
         # Only check the 'pkgs' parameter of pip.install
-        # pylint: disable=unpacking-non-sequence
         (
             mock_install_call_args,
             mock_install_call_kwargs,
         ) = mock_pip_install.call_args
-        # pylint: enable=unpacking-non-sequence
+        
         assert mock_install_call_kwargs["pkgs"] == pkg_to_install

From bb5f719e44309b40ebad755da127062e13e3e064 Mon Sep 17 00:00:00 2001
From: Natalie Lee 
Date: Thu, 25 May 2023 15:26:03 +0000
Subject: [PATCH 108/152] release note on azure extension

---
 doc/topics/releases/templates/3007.0.md.template | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/topics/releases/templates/3007.0.md.template b/doc/topics/releases/templates/3007.0.md.template
index 6a583f94254..ffa50b8aecd 100644
--- a/doc/topics/releases/templates/3007.0.md.template
+++ b/doc/topics/releases/templates/3007.0.md.template
@@ -6,6 +6,9 @@
 Add release specific details below
 -->
 
+## Azure Salt Extension
+
+Starting from Salt version 3007.0, the Azure functionality previously available in the Salt code base has been deprecated. To continue using Salt's features for interacting with Azure resources, users are required to utilize the Azure Salt extension. For more information, refer to the [Azure Salt Extension GitHub repository](https://github.com/salt-extensions/saltext-azurerm).
 
 
-      - id: pip-tools-compile
-        alias: compile-pkg-linux-3.7-zmq-requirements
-        name: Linux Packaging Py3.7 ZeroMQ Requirements
-        files: ^requirements/((base|zeromq|crypto)\.txt|static/pkg/(linux\.in|py3\.7/linux\.txt))$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --platform=linux
-          - --include=requirements/base.txt
-          - --include=requirements/zeromq.txt
-          - requirements/static/pkg/linux.in
-
       - id: pip-tools-compile
         alias: compile-pkg-linux-3.8-zmq-requirements
         name: Linux Packaging Py3.8 ZeroMQ Requirements
@@ -159,19 +146,6 @@ repos:
           - --include=requirements/zeromq.txt
           - requirements/static/pkg/linux.in
 
-      - id: pip-tools-compile
-        alias: compile-pkg-freebsd-3.7-zmq-requirements
-        name: FreeBSD Packaging Py3.7 ZeroMQ Requirements
-        files: ^requirements/((base|zeromq|crypto)\.txt|static/pkg/(freebsd\.in|py3\.7/freebsd\.txt))$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --platform=freebsd
-          - --include=requirements/base.txt
-          - --include=requirements/zeromq.txt
-          - requirements/static/pkg/freebsd.in
-
       - id: pip-tools-compile
         alias: compile-pkg-freebsd-3.8-zmq-requirements
         name: FreeBSD Packaging Py3.8 ZeroMQ Requirements
@@ -260,18 +234,6 @@ repos:
           - --include=requirements/darwin.txt
           - requirements/static/pkg/darwin.in
 
-      - id: pip-tools-compile
-        alias: compile-pkg-windows-3.7-zmq-requirements
-        name: Windows Packaging Py3.7 ZeroMQ Requirements
-        files: ^requirements/((base|zeromq|crypto|windows)\.txt|static/pkg/(windows\.in|py3\.7/windows\.txt))$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --platform=windows
-          - --include=requirements/windows.txt
-          - requirements/static/pkg/windows.in
-
       - id: pip-tools-compile
         alias: compile-pkg-windows-3.8-zmq-requirements
         name: Windows Packaging Py3.8 ZeroMQ Requirements
@@ -323,23 +285,6 @@ repos:
   # <---- Packaging Requirements -------------------------------------------------------------------------------------
 
   # ----- CI Requirements ------------------------------------------------------------------------------------------->
-      - id: pip-tools-compile
-        alias: compile-ci-linux-3.7-zmq-requirements
-        name: Linux CI Py3.7 ZeroMQ Requirements
-        files: ^requirements/((base|zeromq|pytest)\.txt|static/((ci|pkg)/(linux\.in|common\.in)|pkg/py3\.7/linux\.txt))$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --platform=linux
-          - --include=requirements/base.txt
-          - --include=requirements/zeromq.txt
-          - --include=requirements/pytest.txt
-          - --include=requirements/static/pkg/linux.in
-          - --include=requirements/static/ci/common.in
-          - --pip-args=--constraint=requirements/static/pkg/py{py_version}/linux.txt
-          - requirements/static/ci/linux.in
-
       - id: pip-tools-compile
         alias: compile-ci-linux-3.8-zmq-requirements
         name: Linux CI Py3.8 ZeroMQ Requirements
@@ -408,19 +353,6 @@ repos:
           - --pip-args=--constraint=requirements/static/pkg/py{py_version}/linux.txt
           - requirements/static/ci/linux.in
 
-      - id: pip-tools-compile
-        alias: compile-ci-linux-crypto-3.7-requirements
-        name: Linux CI Py3.7 Crypto Requirements
-        files: ^requirements/(crypto\.txt|static/ci/(crypto\.in|py3\.7/linux-crypto\.txt))$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --platform=linux
-          - --out-prefix=linux
-          - --pip-args=--constraint=requirements/static/pkg/py{py_version}/linux.txt
-          - requirements/static/ci/crypto.in
-
       - id: pip-tools-compile
         alias: compile-ci-linux-crypto-3.8-requirements
         name: Linux CI Py3.8 Crypto Requirements
@@ -475,23 +407,6 @@ repos:
           - --pip-args=--constraint=requirements/static/ci/py{py_version}/linux.txt
           - requirements/static/ci/crypto.in
 
-      - id: pip-tools-compile
-        alias: compile-ci-freebsd-3.7-zmq-requirements
-        name: FreeBSD CI Py3.7 ZeroMQ Requirements
-        files: ^requirements/((base|zeromq|pytest)\.txt|static/((ci|pkg)/(freebsd|common)\.in|pkg/py3\.7/freebsd\.txt))$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --platform=freebsd
-          - --include=requirements/base.txt
-          - --include=requirements/zeromq.txt
-          - --include=requirements/pytest.txt
-          - --include=requirements/static/pkg/freebsd.in
-          - --include=requirements/static/ci/common.in
-          - --pip-args=--constraint=requirements/static/pkg/py{py_version}/freebsd.txt
-          - requirements/static/ci/freebsd.in
-
       - id: pip-tools-compile
         alias: compile-ci-freebsd-3.8-zmq-requirements
         name: FreeBSD CI Py3.8 ZeroMQ Requirements
@@ -560,19 +475,6 @@ repos:
           - --pip-args=--constraint=requirements/static/pkg/py{py_version}/freebsd.txt
           - requirements/static/ci/freebsd.in
 
-      - id: pip-tools-compile
-        alias: compile-ci-freebsd-crypto-3.7-requirements
-        name: FreeBSD CI Py3.7 Crypto Requirements
-        files: ^requirements/(crypto\.txt|static/ci/(crypto\.in|py3\.7/freebsd-crypto\.txt))$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --platform=freebsd
-          - --out-prefix=freebsd
-          - --pip-args=--constraint=requirements/static/ci/py{py_version}/freebsd.txt
-          - requirements/static/ci/crypto.in
-
       - id: pip-tools-compile
         alias: compile-ci-freebsd-crypto-3.8-requirements
         name: FreeBSD CI Py3.8 Crypto Requirements
@@ -714,22 +616,6 @@ repos:
           - --pip-args=--constraint=requirements/static/ci/py{py_version}/darwin.txt
           - requirements/static/ci/crypto.in
 
-      - id: pip-tools-compile
-        alias: compile-ci-windows-3.7-zmq-requirements
-        name: Windows CI Py3.7 ZeroMQ Requirements
-        files: requirements/((base|zeromq|pytest)\.txt|static/((ci|pkg)/(windows|common)\.in|pkg/py3\.7/windows\.txt))$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --platform=windows
-          - --include=requirements/windows.txt
-          - --include=requirements/pytest.txt
-          - --include=requirements/static/pkg/windows.in
-          - --include=requirements/static/ci/common.in
-          - --pip-args=--constraint=requirements/static/pkg/py{py_version}/windows.txt
-          - requirements/static/ci/windows.in
-
       - id: pip-tools-compile
         alias: compile-ci-windows-3.8-zmq-requirements
         name: Windows CI Py3.8 ZeroMQ Requirements
@@ -794,19 +680,6 @@ repos:
           - --pip-args=--constraint=requirements/static/pkg/py{py_version}/windows.txt
           - requirements/static/ci/windows.in
 
-      - id: pip-tools-compile
-        alias: compile-ci-windows-crypto-3.7-requirements
-        name: Windows CI Py3.7 Crypto Requirements
-        files: ^requirements/(crypto\.txt|static/ci/(crypto\.in|py3\.7/windows-crypto\.txt))$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --platform=windows
-          - --out-prefix=windows
-          - --pip-args=--constraint=requirements/static/ci/py{py_version}/windows.txt
-          - requirements/static/ci/crypto.in
-
       - id: pip-tools-compile
         alias: compile-ci-windows-crypto-3.8-requirements
         name: Windows CI Py3.8 Crypto Requirements
@@ -863,22 +736,6 @@ repos:
 
 
   # ----- Cloud CI Requirements ------------------------------------------------------------------------------------->
-      - id: pip-tools-compile
-        alias: compile-ci-cloud-3.7-requirements
-        name: Cloud CI Py3.7 Requirements
-        files: ^requirements/((base|zeromq|pytest)\.txt|static/(pkg/linux\.in|ci/((cloud|common)\.in|py3\.7/cloud\.txt)))$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --include=requirements/base.txt
-          - --include=requirements/zeromq.txt
-          - --include=requirements/pytest.txt
-          - --include=requirements/static/pkg/linux.in
-          - --include=requirements/static/ci/common.in
-          - --pip-args=--constraint=requirements/static/ci/py{py_version}/linux.txt
-          - requirements/static/ci/cloud.in
-
       - id: pip-tools-compile
         alias: compile-ci-cloud-3.8-requirements
         name: Cloud CI Py3.8 Requirements
@@ -929,19 +786,6 @@ repos:
   # <---- Cloud CI Requirements --------------------------------------------------------------------------------------
 
   # ----- Doc CI Requirements --------------------------------------------------------------------------------------->
-      - id: pip-tools-compile
-        alias: compile-doc-requirements
-        name: Docs CI Py3.7 Requirements
-        files: ^requirements/((base|zeromq|pytest)\.txt|static/ci/(docs|common|linux)\.in|static/pkg/linux\.in|static/pkg/.*/linux\.txt)$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --platform=linux
-          - --include=requirements/base.txt
-          - --include=requirements/zeromq.txt
-          - requirements/static/ci/docs.in
-
       - id: pip-tools-compile
         alias: compile-doc-requirements
         name: Docs CI Py3.8 Requirements
@@ -984,23 +828,6 @@ repos:
   # <---- Doc CI Requirements ----------------------------------------------------------------------------------------
 
   # ----- Lint CI Requirements -------------------------------------------------------------------------------------->
-      - id: pip-tools-compile
-        alias: compile-ci-lint-3.7-requirements
-        name: Lint CI Py3.7 Requirements
-        files: ^requirements/((base|zeromq)\.txt|static/(pkg/linux\.in|ci/(linux\.in|common\.in|lint\.in|py3\.7/linux\.txt)))$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --platform=linux
-          - --include=requirements/base.txt
-          - --include=requirements/zeromq.txt
-          - --include=requirements/static/pkg/linux.in
-          - --include=requirements/static/ci/linux.in
-          - --include=requirements/static/ci/common.in
-          - --pip-args=--constraint=requirements/static/ci/py{py_version}/linux.txt
-          - requirements/static/ci/lint.in
-
       - id: pip-tools-compile
         alias: compile-ci-lint-3.8-requirements
         name: Lint CI Py3.8 Requirements
@@ -1093,17 +920,6 @@ repos:
   # <---- Changelog --------------------------------------------------------------------------------------------------
 
   # ----- Invoke ---------------------------------------------------------------------------------------------------->
-      - id: pip-tools-compile
-        alias: compile-ci-invoke-3.7-requirements
-        name: Linux CI Py3.7 Invoke Requirements
-        files: ^requirements/static/ci/(invoke\.in|py3.7/(invoke|linux)\.txt)$
-        pass_filenames: false
-        args:
-          - -v
-          - --py-version=3.7
-          - --pip-args=--constraint=requirements/static/ci/py{py_version}/linux.txt
-          - requirements/static/ci/invoke.in
-
       - id: pip-tools-compile
         alias: compile-ci-invoke-3.8-requirements
         name: Linux CI Py3.8 Invoke Requirements
@@ -1197,8 +1013,8 @@ repos:
     rev: v2.37.2
     hooks:
       - id: pyupgrade
-        name: Drop six usage and Py2 support
-        args: [--py3-plus, --keep-mock]
+        name: Upgrade code for Py3.8+
+        args: [--py38-plus, --keep-mock]
         exclude: >
           (?x)^(
             salt/client/ssh/ssh_py_shim.py
diff --git a/.rstcheck.cfg b/.rstcheck.cfg
deleted file mode 100644
index a2c5731b9e1..00000000000
--- a/.rstcheck.cfg
+++ /dev/null
@@ -1,27 +0,0 @@
-[rstcheck]
-ignore_directives=
-  automodule,
-  autoclass,
-  autofunction,
-  conf_proxy,
-  conf_log,
-  conf_master,
-  conf_minion,
-  releasestree,
-  jinja_ref,
-  salt:event
-ignore_roles=
-  conf_master,
-  conf_minion,
-  conf_proxy,
-  conf_log,
-  formula_url,
-  issue,
-  pull,
-  blob,
-  jinja_ref
-ignore_substitutions=
-  saltrepo,
-  repo_primary_branch,
-  windownload,
-  osxdownloadpy3
diff --git a/noxfile.py b/noxfile.py
index 9a7afa93bdc..30687acf349 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -104,7 +104,7 @@ def session_warn(session, message):
     try:
         session.warn(message)
     except AttributeError:
-        session.log("WARNING: {}".format(message))
+        session.log(f"WARNING: {message}")
 
 
 def session_run_always(session, *command, **kwargs):
@@ -129,15 +129,15 @@ def session_run_always(session, *command, **kwargs):
 
 def find_session_runner(session, name, python_version, onedir=False, **kwargs):
     if onedir:
-        name += "-onedir-{}".format(ONEDIR_PYTHON_PATH)
+        name += f"-onedir-{ONEDIR_PYTHON_PATH}"
     else:
-        name += "-{}".format(python_version)
+        name += f"-{python_version}"
     for s, _ in session._runner.manifest.list_all_sessions():
         if name not in s.signatures:
             continue
         for signature in s.signatures:
             for key, value in kwargs.items():
-                param = "{}={!r}".format(key, value)
+                param = f"{key}={value!r}"
                 if param not in signature:
                     break
             else:
@@ -185,10 +185,8 @@ def _get_session_python_version_info(session):
 
 def _get_pydir(session):
     version_info = _get_session_python_version_info(session)
-    if version_info < (3, 5):
-        session.error("Only Python >= 3.5 is supported")
-    if IS_WINDOWS and version_info < (3, 6):
-        session.error("Only Python >= 3.6 is supported on Windows")
+    if version_info < (3, 8):
+        session.error("Only Python >= 3.8 is supported")
     return "py{}.{}".format(*version_info)
 
 
@@ -203,7 +201,7 @@ def _get_pip_requirements_file(session, transport, crypto=None, requirements_typ
                 "static",
                 requirements_type,
                 pydir,
-                "{}-windows.txt".format(transport),
+                f"{transport}-windows.txt",
             )
             if os.path.exists(_requirements_file):
                 return _requirements_file
@@ -217,7 +215,7 @@ def _get_pip_requirements_file(session, transport, crypto=None, requirements_typ
         )
         if os.path.exists(_requirements_file):
             return _requirements_file
-        session.error("Could not find a windows requirements file for {}".format(pydir))
+        session.error(f"Could not find a windows requirements file for {pydir}")
     elif IS_DARWIN:
         if crypto is None:
             _requirements_file = os.path.join(
@@ -225,7 +223,7 @@ def _get_pip_requirements_file(session, transport, crypto=None, requirements_typ
                 "static",
                 requirements_type,
                 pydir,
-                "{}-darwin.txt".format(transport),
+                f"{transport}-darwin.txt",
             )
             if os.path.exists(_requirements_file):
                 return _requirements_file
@@ -239,7 +237,7 @@ def _get_pip_requirements_file(session, transport, crypto=None, requirements_typ
         )
         if os.path.exists(_requirements_file):
             return _requirements_file
-        session.error("Could not find a darwin requirements file for {}".format(pydir))
+        session.error(f"Could not find a darwin requirements file for {pydir}")
     elif IS_FREEBSD:
         if crypto is None:
             _requirements_file = os.path.join(
@@ -247,7 +245,7 @@ def _get_pip_requirements_file(session, transport, crypto=None, requirements_typ
                 "static",
                 requirements_type,
                 pydir,
-                "{}-freebsd.txt".format(transport),
+                f"{transport}-freebsd.txt",
             )
             if os.path.exists(_requirements_file):
                 return _requirements_file
@@ -261,7 +259,7 @@ def _get_pip_requirements_file(session, transport, crypto=None, requirements_typ
         )
         if os.path.exists(_requirements_file):
             return _requirements_file
-        session.error("Could not find a freebsd requirements file for {}".format(pydir))
+        session.error(f"Could not find a freebsd requirements file for {pydir}")
     else:
         if crypto is None:
             _requirements_file = os.path.join(
@@ -269,7 +267,7 @@ def _get_pip_requirements_file(session, transport, crypto=None, requirements_typ
                 "static",
                 requirements_type,
                 pydir,
-                "{}-linux.txt".format(transport),
+                f"{transport}-linux.txt",
             )
             if os.path.exists(_requirements_file):
                 return _requirements_file
@@ -283,7 +281,7 @@ def _get_pip_requirements_file(session, transport, crypto=None, requirements_typ
         )
         if os.path.exists(_requirements_file):
             return _requirements_file
-        session.error("Could not find a linux requirements file for {}".format(pydir))
+        session.error(f"Could not find a linux requirements file for {pydir}")
 
 
 def _upgrade_pip_setuptools_and_wheel(session, upgrade=True, onedir=False):
@@ -571,7 +569,7 @@ def test_parametrized(session, coverage, transport, crypto):
             session.install(*install_command, silent=PIP_INSTALL_SILENT)
 
     cmd_args = [
-        "--transport={}".format(transport),
+        f"--transport={transport}",
     ] + session.posargs
     _pytest(session, coverage=coverage, cmd_args=cmd_args)
 
@@ -1016,7 +1014,7 @@ def _pytest(session, coverage, cmd_args, env=None):
         if arg == "--log-file" or arg.startswith("--log-file="):
             break
     else:
-        args.append("--log-file={}".format(RUNTESTS_LOGFILE))
+        args.append(f"--log-file={RUNTESTS_LOGFILE}")
     args.extend(cmd_args)
 
     if PRINT_SYSTEM_INFO and "--sysinfo" not in args:
@@ -1358,7 +1356,7 @@ def _lint(
         session.run("pylint", "--version")
         pylint_report_path = os.environ.get("PYLINT_REPORT")
 
-    cmd_args = ["pylint", "--rcfile={}".format(rcfile)] + list(flags) + list(paths)
+    cmd_args = ["pylint", f"--rcfile={rcfile}"] + list(flags) + list(paths)
 
     cmd_kwargs = {"env": {"PYTHONUNBUFFERED": "1"}}
 
@@ -1433,8 +1431,8 @@ def lint(session):
     """
     Run PyLint against Salt and it's test suite. Set PYLINT_REPORT to a path to capture output.
     """
-    session.notify("lint-salt-{}".format(session.python))
-    session.notify("lint-tests-{}".format(session.python))
+    session.notify(f"lint-salt-{session.python}")
+    session.notify(f"lint-tests-{session.python}")
 
 
 @nox.session(python="3", name="lint-salt")
@@ -1498,7 +1496,7 @@ def docs(session, compress, update, clean):
     """
     Build Salt's Documentation
     """
-    session.notify("docs-html-{}(compress={})".format(session.python, compress))
+    session.notify(f"docs-html-{session.python}(compress={compress})")
     session.notify(
         find_session_runner(
             session,
diff --git a/requirements/static/ci/py3.7/cloud.txt b/requirements/static/ci/py3.7/cloud.txt
deleted file mode 100644
index 956b961aa19..00000000000
--- a/requirements/static/ci/py3.7/cloud.txt
+++ /dev/null
@@ -1,538 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/ci/py3.7/cloud.txt --pip-args='--constraint=requirements/static/ci/py3.7/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/cloud.in requirements/static/ci/common.in requirements/static/pkg/linux.in requirements/zeromq.txt
-#
-aiohttp==3.8.1
-    # via etcd3-py
-aiosignal==1.2.0
-    # via aiohttp
-apache-libcloud==2.5.0 ; sys_platform != "win32"
-    # via
-    #   -r requirements/static/ci/cloud.in
-    #   -r requirements/static/ci/common.in
-asn1crypto==1.4.0
-    # via
-    #   certvalidator
-    #   oscrypto
-async-timeout==4.0.2
-    # via aiohttp
-asynctest==0.13.0
-    # via aiohttp
-attrs==21.2.0
-    # via
-    #   aiohttp
-    #   jsonschema
-    #   pytest
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-skip-markers
-    #   pytest-system-statistics
-backports.entry-points-selectable==1.1.0
-    # via virtualenv
-bcrypt==3.2.0
-    # via
-    #   paramiko
-    #   passlib
-boto3==1.21.46 ; python_version >= "3.6"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   moto
-boto==2.49.0
-    # via -r requirements/static/ci/common.in
-botocore==1.24.46
-    # via
-    #   boto3
-    #   moto
-    #   s3transfer
-cachetools==4.2.2
-    # via google-auth
-cassandra-driver==3.25.0
-    # via -r requirements/static/ci/common.in
-certifi==2022.12.7
-    # via
-    #   -r requirements/static/ci/common.in
-    #   kubernetes
-    #   requests
-certvalidator==0.11.1
-    # via vcert
-cffi==1.14.6
-    # via
-    #   -r requirements/static/ci/common.in
-    #   bcrypt
-    #   cryptography
-    #   napalm
-    #   pynacl
-charset-normalizer==2.0.12
-    # via
-    #   aiohttp
-    #   requests
-cheetah3==3.2.6.post1
-    # via -r requirements/static/ci/common.in
-cheroot==8.5.2
-    # via cherrypy
-cherrypy==18.6.1
-    # via
-    #   -r requirements/static/ci/common.in
-    #   -r requirements/static/pkg/linux.in
-ciscoconfparse==1.5.46
-    # via napalm
-click==8.0.1
-    # via geomet
-clustershell==1.8.3
-    # via -r requirements/static/ci/common.in
-colorama==0.4.4
-    # via ciscoconfparse
-contextvars==2.4
-    # via -r requirements/base.txt
-croniter==1.0.15 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-cryptography==39.0.2
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   etcd3-py
-    #   moto
-    #   paramiko
-    #   pyopenssl
-    #   pyspnego
-    #   requests-ntlm
-    #   smbprotocol
-    #   vcert
-distlib==0.3.3
-    # via virtualenv
-distro==1.6.0
-    # via
-    #   -r requirements/base.txt
-    #   pytest-skip-markers
-dnspython==2.1.0
-    # via
-    #   -r requirements/static/ci/common.in
-    #   ciscoconfparse
-    #   python-etcd
-docker==5.0.2
-    # via
-    #   -r requirements/static/ci/common.in
-    #   pytest-salt-factories
-etcd3-py==0.1.6 ; python_version >= "3.6"
-    # via -r requirements/static/ci/common.in
-exceptiongroup==1.0.4
-    # via pytest
-filelock==3.0.12
-    # via virtualenv
-flaky==3.7.0
-    # via -r requirements/pytest.txt
-frozenlist==1.3.0
-    # via
-    #   aiohttp
-    #   aiosignal
-future==0.18.3
-    # via
-    #   napalm
-    #   textfsm
-genshi==0.7.5
-    # via -r requirements/static/ci/common.in
-geomet==0.2.1.post1
-    # via cassandra-driver
-gitdb==4.0.7
-    # via gitpython
-gitpython==3.1.30 ; python_version >= "3.7"
-    # via -r requirements/static/ci/common.in
-google-auth==2.1.0
-    # via kubernetes
-idna==2.8
-    # via
-    #   etcd3-py
-    #   requests
-    #   yarl
-immutables==0.16
-    # via contextvars
-importlib-metadata==4.8.1
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   backports.entry-points-selectable
-    #   click
-    #   jsonschema
-    #   mako
-    #   moto
-    #   pluggy
-    #   pytest
-    #   virtualenv
-iniconfig==1.1.1
-    # via pytest
-ipaddress==1.0.23
-    # via kubernetes
-jaraco.classes==3.2.1
-    # via jaraco.collections
-jaraco.collections==3.4.0
-    # via cherrypy
-jaraco.functools==3.3.0
-    # via
-    #   cheroot
-    #   jaraco.text
-    #   tempora
-jaraco.text==3.5.1
-    # via jaraco.collections
-jinja2==3.1.2
-    # via
-    #   -r requirements/base.txt
-    #   junos-eznc
-    #   moto
-    #   napalm
-jmespath==0.10.0
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/static/ci/common.in
-    #   boto3
-    #   botocore
-jsonschema==3.2.0
-    # via -r requirements/static/ci/common.in
-junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   napalm
-jxmlease==1.0.3 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin"
-    # via -r requirements/static/ci/common.in
-keyring==5.7.1
-    # via -r requirements/static/ci/common.in
-kubernetes==3.0.0
-    # via -r requirements/static/ci/common.in
-libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin"
-    # via -r requirements/static/ci/common.in
-loguru==0.6.0
-    # via ciscoconfparse
-looseversion==1.0.2
-    # via -r requirements/base.txt
-lxml==4.9.1
-    # via
-    #   junos-eznc
-    #   napalm
-    #   ncclient
-mako==1.2.2
-    # via -r requirements/static/ci/common.in
-markupsafe==2.1.2
-    # via
-    #   -r requirements/base.txt
-    #   jinja2
-    #   mako
-    #   moto
-    #   werkzeug
-mock==4.0.3
-    # via -r requirements/pytest.txt
-more-itertools==8.8.0
-    # via
-    #   cheroot
-    #   cherrypy
-    #   jaraco.classes
-    #   jaraco.functools
-moto==3.0.1 ; python_version >= "3.6"
-    # via -r requirements/static/ci/common.in
-msgpack==1.0.2
-    # via
-    #   -r requirements/base.txt
-    #   pytest-salt-factories
-multidict==6.0.2
-    # via
-    #   aiohttp
-    #   yarl
-napalm==3.3.1 ; sys_platform != "win32" and python_version > "3.6" and python_version < "3.10"
-    # via -r requirements/static/ci/common.in
-ncclient==0.6.12
-    # via
-    #   junos-eznc
-    #   napalm
-netaddr==0.7.19
-    # via
-    #   -r requirements/static/ci/cloud.in
-    #   junos-eznc
-    #   napalm
-    #   pyeapi
-netmiko==3.4.0
-    # via napalm
-ntc-templates==2.3.2
-    # via
-    #   junos-eznc
-    #   netmiko
-ntlm-auth==1.3.0
-    # via requests-ntlm
-oscrypto==1.2.1
-    # via certvalidator
-packaging==21.3
-    # via
-    #   -r requirements/base.txt
-    #   pytest
-paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   junos-eznc
-    #   napalm
-    #   ncclient
-    #   netmiko
-    #   scp
-passlib[bcrypt]==1.7.4
-    # via
-    #   -r requirements/static/ci/common.in
-    #   ciscoconfparse
-platformdirs==2.3.0
-    # via virtualenv
-pluggy==1.0.0
-    # via pytest
-portend==2.7.1
-    # via cherrypy
-profitbricks==4.1.3
-    # via -r requirements/static/ci/cloud.in
-psutil==5.8.0
-    # via
-    #   -r requirements/base.txt
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-system-statistics
-pyasn1-modules==0.2.8
-    # via google-auth
-pyasn1==0.4.8
-    # via
-    #   pyasn1-modules
-    #   rsa
-pycparser==2.19
-    # via cffi
-pycryptodomex==3.10.1
-    # via -r requirements/crypto.txt
-pyeapi==0.8.4
-    # via napalm
-pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd"
-    # via -r requirements/static/ci/common.in
-pynacl==1.4.0
-    # via paramiko
-pyopenssl==23.0.0
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   etcd3-py
-pyparsing==3.0.9
-    # via
-    #   junos-eznc
-    #   packaging
-pypsexec==0.1.0
-    # via -r requirements/static/ci/cloud.in
-pyrsistent==0.18.0
-    # via jsonschema
-pyserial==3.5
-    # via
-    #   junos-eznc
-    #   netmiko
-pyspnego==0.8.0
-    # via
-    #   -r requirements/static/ci/cloud.in
-    #   smbprotocol
-pytest-custom-exit-code==0.3.0
-    # via -r requirements/pytest.txt
-pytest-helpers-namespace==2021.4.29
-    # via
-    #   -r requirements/pytest.txt
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-pytest-httpserver==1.0.4
-    # via -r requirements/pytest.txt
-pytest-salt-factories[docker]==1.0.0rc21 ; sys_platform != "win32"
-    # via -r requirements/pytest.txt
-pytest-shell-utilities==1.6.0
-    # via pytest-salt-factories
-pytest-skip-markers==1.2.0
-    # via
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-system-statistics
-pytest-subtests==0.5.0
-    # via -r requirements/pytest.txt
-pytest-system-statistics==1.0.2
-    # via pytest-salt-factories
-pytest-tempdir==2019.10.12
-    # via
-    #   -r requirements/pytest.txt
-    #   pytest-salt-factories
-pytest-timeout==2.0.2
-    # via -r requirements/pytest.txt
-pytest==7.2.0 ; python_version > "3.6"
-    # via
-    #   -r requirements/pytest.txt
-    #   pytest-custom-exit-code
-    #   pytest-helpers-namespace
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-skip-markers
-    #   pytest-subtests
-    #   pytest-system-statistics
-    #   pytest-tempdir
-    #   pytest-timeout
-python-dateutil==2.8.2
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   botocore
-    #   croniter
-    #   kubernetes
-    #   moto
-    #   vcert
-python-etcd==0.4.5
-    # via -r requirements/static/ci/common.in
-python-gnupg==0.4.8
-    # via -r requirements/static/pkg/linux.in
-pytz==2022.1
-    # via
-    #   moto
-    #   tempora
-pyvmomi==7.0.2
-    # via -r requirements/static/ci/common.in
-pywinrm==0.3.0
-    # via -r requirements/static/ci/cloud.in
-pyyaml==5.4.1
-    # via
-    #   -r requirements/base.txt
-    #   clustershell
-    #   junos-eznc
-    #   kubernetes
-    #   napalm
-    #   yamlordereddictloader
-pyzmq==23.2.0 ; python_version < "3.11"
-    # via
-    #   -r requirements/zeromq.txt
-    #   pytest-salt-factories
-requests-ntlm==1.1.0
-    # via pywinrm
-requests==2.31.0
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/static/ci/common.in
-    #   apache-libcloud
-    #   docker
-    #   etcd3-py
-    #   kubernetes
-    #   moto
-    #   napalm
-    #   profitbricks
-    #   pyvmomi
-    #   pywinrm
-    #   requests-ntlm
-    #   responses
-    #   vcert
-responses==0.14.0
-    # via moto
-rfc3987==1.3.8
-    # via -r requirements/static/ci/common.in
-rpm-vercmp==0.1.2
-    # via -r requirements/static/pkg/linux.in
-rsa==4.7.2
-    # via google-auth
-s3transfer==0.5.0
-    # via boto3
-scp==0.14.1
-    # via
-    #   junos-eznc
-    #   napalm
-    #   netmiko
-semantic-version==2.9.0
-    # via etcd3-py
-setproctitle==1.3.2
-    # via -r requirements/static/pkg/linux.in
-six==1.16.0
-    # via
-    #   bcrypt
-    #   cassandra-driver
-    #   cheroot
-    #   etcd3-py
-    #   genshi
-    #   geomet
-    #   jsonschema
-    #   junos-eznc
-    #   kazoo
-    #   kubernetes
-    #   ncclient
-    #   paramiko
-    #   profitbricks
-    #   pynacl
-    #   pypsexec
-    #   python-dateutil
-    #   pyvmomi
-    #   pywinrm
-    #   responses
-    #   textfsm
-    #   transitions
-    #   vcert
-    #   virtualenv
-    #   websocket-client
-smbprotocol==1.10.1
-    # via
-    #   -r requirements/static/ci/cloud.in
-    #   pypsexec
-smmap==4.0.0
-    # via gitdb
-sqlparse==0.4.4
-    # via -r requirements/static/ci/common.in
-strict-rfc3339==0.7
-    # via -r requirements/static/ci/common.in
-tempora==4.1.1
-    # via portend
-tenacity==8.0.1
-    # via netmiko
-textfsm==1.1.2
-    # via
-    #   napalm
-    #   ntc-templates
-timelib==0.2.5
-    # via -r requirements/static/pkg/linux.in
-toml==0.10.2
-    # via -r requirements/static/ci/common.in
-tomli==2.0.1
-    # via pytest
-tornado==6.2.0 ; python_version < "3.8"
-    # via -r requirements/base.txt
-transitions==0.8.9
-    # via junos-eznc
-typing-extensions==3.10.0.0
-    # via
-    #   aiohttp
-    #   async-timeout
-    #   gitpython
-    #   immutables
-    #   importlib-metadata
-    #   pytest-shell-utilities
-    #   pytest-system-statistics
-    #   yarl
-urllib3==1.26.6
-    # via
-    #   botocore
-    #   kubernetes
-    #   python-etcd
-    #   requests
-    #   responses
-vcert==0.7.4 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-virtualenv==20.8.0
-    # via
-    #   -r requirements/static/ci/common.in
-    #   pytest-salt-factories
-watchdog==2.1.5
-    # via -r requirements/static/ci/common.in
-websocket-client==0.40.0
-    # via
-    #   docker
-    #   kubernetes
-werkzeug==2.2.3
-    # via
-    #   moto
-    #   pytest-httpserver
-xmltodict==0.12.0
-    # via
-    #   moto
-    #   pywinrm
-yamlordereddictloader==0.4.0
-    # via junos-eznc
-yarl==1.7.2
-    # via aiohttp
-zc.lockfile==2.0
-    # via cherrypy
-zipp==3.5.0
-    # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/requirements/static/ci/py3.7/docs.txt b/requirements/static/ci/py3.7/docs.txt
deleted file mode 100644
index 3e1427f5149..00000000000
--- a/requirements/static/ci/py3.7/docs.txt
+++ /dev/null
@@ -1,214 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/ci/py3.7/docs.txt requirements/base.txt requirements/static/ci/docs.in requirements/zeromq.txt
-#
-alabaster==0.7.12
-    # via sphinx
-babel==2.9.1
-    # via sphinx
-certifi==2022.12.7
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   requests
-charset-normalizer==2.0.12
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   requests
-cheroot==8.5.2
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   cherrypy
-cherrypy==18.6.1
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/static/ci/docs.in
-contextvars==2.4
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/base.txt
-distro==1.5.0
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/base.txt
-docutils==0.16
-    # via sphinx
-idna==2.8
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   requests
-imagesize==1.2.0
-    # via sphinx
-immutables==0.15
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   contextvars
-importlib-metadata==4.6.4
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   sphinxcontrib-spelling
-jaraco.classes==3.2.1
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   jaraco.collections
-jaraco.collections==3.4.0
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   cherrypy
-jaraco.functools==2.0
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   cheroot
-    #   jaraco.text
-    #   tempora
-jaraco.text==3.5.1
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   jaraco.collections
-jinja2==3.1.2
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/base.txt
-    #   myst-docutils
-    #   sphinx
-jmespath==1.0.1
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/base.txt
-linkify-it-py==1.0.3
-    # via myst-docutils
-looseversion==1.0.2
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/base.txt
-markdown-it-py==2.2.0
-    # via
-    #   mdit-py-plugins
-    #   myst-docutils
-markupsafe==2.1.2
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/base.txt
-    #   jinja2
-mdit-py-plugins==0.3.3
-    # via myst-docutils
-mdurl==0.1.2
-    # via markdown-it-py
-more-itertools==5.0.0
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   cheroot
-    #   cherrypy
-    #   jaraco.classes
-    #   jaraco.functools
-msgpack==1.0.2
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/base.txt
-myst-docutils[linkify]==0.18.1
-    # via -r requirements/static/ci/docs.in
-packaging==21.3
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/base.txt
-    #   sphinx
-portend==2.4
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   cherrypy
-psutil==5.8.0
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/base.txt
-pycryptodomex==3.9.8
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/crypto.txt
-pyenchant==3.2.2
-    # via sphinxcontrib-spelling
-pygments==2.8.1
-    # via sphinx
-pyparsing==3.0.9
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   packaging
-pytz==2022.1
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   babel
-    #   tempora
-pyyaml==5.4.1
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/base.txt
-    #   myst-docutils
-pyzmq==23.2.0 ; python_version < "3.11"
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/zeromq.txt
-requests==2.31.0
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/base.txt
-    #   sphinx
-six==1.16.0
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   cheroot
-    #   more-itertools
-    #   sphinxcontrib.httpdomain
-snowballstemmer==2.1.0
-    # via sphinx
-sphinx==3.5.2 ; python_version < "3.9"
-    # via
-    #   -r requirements/static/ci/docs.in
-    #   sphinxcontrib-spelling
-    #   sphinxcontrib.httpdomain
-sphinxcontrib-applehelp==1.0.2
-    # via sphinx
-sphinxcontrib-devhelp==1.0.2
-    # via sphinx
-sphinxcontrib-htmlhelp==1.0.3
-    # via sphinx
-sphinxcontrib-jsmath==1.0.1
-    # via sphinx
-sphinxcontrib-qthelp==1.0.3
-    # via sphinx
-sphinxcontrib-serializinghtml==1.1.4
-    # via sphinx
-sphinxcontrib-spelling==7.7.0
-    # via -r requirements/static/ci/docs.in
-sphinxcontrib.httpdomain==1.8.1
-    # via -r requirements/static/ci/docs.in
-tempora==4.1.1
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   portend
-tornado==6.2.0 ; python_version < "3.8"
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   -r requirements/base.txt
-typing-extensions==3.10.0.0
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   importlib-metadata
-    #   markdown-it-py
-    #   myst-docutils
-uc-micro-py==1.0.1
-    # via linkify-it-py
-urllib3==1.26.6
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   requests
-zc.lockfile==1.4
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   cherrypy
-zipp==3.5.0
-    # via
-    #   -c requirements/static/ci/py3.7/linux.txt
-    #   importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/requirements/static/ci/py3.7/freebsd-crypto.txt b/requirements/static/ci/py3.7/freebsd-crypto.txt
deleted file mode 100644
index 7c8974d75e9..00000000000
--- a/requirements/static/ci/py3.7/freebsd-crypto.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/ci/py3.7/freebsd-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.7/freebsd.txt' requirements/static/ci/crypto.in
-#
-m2crypto==0.38.0
-    # via -r requirements/static/ci/crypto.in
-pycryptodome==3.9.7
-    # via -r requirements/static/ci/crypto.in
diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt
deleted file mode 100644
index b400a1450ff..00000000000
--- a/requirements/static/ci/py3.7/freebsd.txt
+++ /dev/null
@@ -1,513 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/ci/py3.7/freebsd.txt --pip-args='--constraint=requirements/static/pkg/py3.7/freebsd.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/freebsd.in requirements/static/pkg/freebsd.in requirements/zeromq.txt
-#
-aiohttp==3.8.1
-    # via etcd3-py
-aiosignal==1.2.0
-    # via aiohttp
-apache-libcloud==2.5.0 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-asn1crypto==1.3.0
-    # via
-    #   certvalidator
-    #   oscrypto
-async-timeout==4.0.2
-    # via aiohttp
-asynctest==0.13.0
-    # via aiohttp
-attrs==20.3.0
-    # via
-    #   aiohttp
-    #   jsonschema
-    #   pytest
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-skip-markers
-    #   pytest-system-statistics
-backports.entry-points-selectable==1.1.0
-    # via virtualenv
-bcrypt==3.1.6
-    # via
-    #   paramiko
-    #   passlib
-boto3==1.21.46 ; python_version >= "3.6"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   moto
-boto==2.49.0
-    # via -r requirements/static/ci/common.in
-botocore==1.24.46
-    # via
-    #   boto3
-    #   moto
-    #   s3transfer
-cachetools==3.1.0
-    # via google-auth
-cassandra-driver==3.24.0
-    # via -r requirements/static/ci/common.in
-certifi==2022.12.7
-    # via
-    #   -r requirements/static/ci/common.in
-    #   kubernetes
-    #   requests
-certvalidator==0.11.1
-    # via vcert
-cffi==1.14.6
-    # via
-    #   -r requirements/static/ci/common.in
-    #   bcrypt
-    #   cryptography
-    #   napalm
-    #   pygit2
-    #   pynacl
-charset-normalizer==2.0.12
-    # via
-    #   aiohttp
-    #   requests
-cheetah3==3.2.6.post2
-    # via -r requirements/static/ci/common.in
-cheroot==8.5.2
-    # via cherrypy
-cherrypy==18.6.1
-    # via
-    #   -r requirements/static/ci/common.in
-    #   -r requirements/static/pkg/freebsd.in
-ciscoconfparse==1.5.19
-    # via napalm
-click==7.1.2
-    # via geomet
-clustershell==1.8.3
-    # via -r requirements/static/ci/common.in
-colorama==0.4.3
-    # via ciscoconfparse
-contextvars==2.4
-    # via -r requirements/base.txt
-croniter==0.3.29 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-cryptography==39.0.2
-    # via
-    #   etcd3-py
-    #   moto
-    #   paramiko
-    #   pyopenssl
-    #   vcert
-distlib==0.3.2
-    # via virtualenv
-distro==1.5.0
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/static/pkg/freebsd.in
-    #   pytest-skip-markers
-dnspython==1.16.0
-    # via
-    #   -r requirements/static/ci/common.in
-    #   ciscoconfparse
-    #   python-etcd
-docker==5.0.3
-    # via
-    #   -r requirements/static/ci/common.in
-    #   pytest-salt-factories
-etcd3-py==0.1.6 ; python_version >= "3.6"
-    # via -r requirements/static/ci/common.in
-exceptiongroup==1.0.4
-    # via pytest
-filelock==3.0.12
-    # via virtualenv
-flaky==3.7.0
-    # via -r requirements/pytest.txt
-frozenlist==1.3.0
-    # via
-    #   aiohttp
-    #   aiosignal
-future==0.18.3
-    # via
-    #   napalm
-    #   textfsm
-genshi==0.7.5
-    # via -r requirements/static/ci/common.in
-geomet==0.2.1.post1
-    # via cassandra-driver
-gitdb==4.0.5
-    # via gitpython
-gitpython==3.1.30 ; python_version >= "3.7"
-    # via -r requirements/static/ci/common.in
-google-auth==1.6.3
-    # via kubernetes
-hglib==2.6.1
-    # via -r requirements/static/ci/freebsd.in
-idna==2.8
-    # via
-    #   etcd3-py
-    #   requests
-    #   yarl
-immutables==0.15
-    # via contextvars
-importlib-metadata==4.6.4
-    # via
-    #   -r requirements/static/pkg/freebsd.in
-    #   backports.entry-points-selectable
-    #   jsonschema
-    #   mako
-    #   moto
-    #   pluggy
-    #   pytest
-    #   virtualenv
-iniconfig==1.0.1
-    # via pytest
-ipaddress==1.0.22
-    # via kubernetes
-jaraco.classes==3.2.1
-    # via jaraco.collections
-jaraco.collections==3.4.0
-    # via cherrypy
-jaraco.functools==2.0
-    # via
-    #   cheroot
-    #   jaraco.text
-    #   tempora
-jaraco.text==3.5.1
-    # via jaraco.collections
-jinja2==3.1.2
-    # via
-    #   -r requirements/base.txt
-    #   junos-eznc
-    #   moto
-    #   napalm
-jmespath==1.0.1
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/static/ci/common.in
-    #   boto3
-    #   botocore
-jsonschema==3.2.0
-    # via -r requirements/static/ci/common.in
-junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   napalm
-jxmlease==1.0.1 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin"
-    # via -r requirements/static/ci/common.in
-keyring==5.7.1
-    # via -r requirements/static/ci/common.in
-kubernetes==3.0.0
-    # via -r requirements/static/ci/common.in
-libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin"
-    # via -r requirements/static/ci/common.in
-looseversion==1.0.2
-    # via -r requirements/base.txt
-lxml==4.9.1
-    # via
-    #   junos-eznc
-    #   napalm
-    #   ncclient
-mako==1.2.2
-    # via -r requirements/static/ci/common.in
-markupsafe==2.1.2
-    # via
-    #   -r requirements/base.txt
-    #   jinja2
-    #   mako
-    #   moto
-    #   werkzeug
-mercurial==6.0.1
-    # via -r requirements/static/ci/freebsd.in
-mock==3.0.5
-    # via -r requirements/pytest.txt
-more-itertools==5.0.0
-    # via
-    #   cheroot
-    #   cherrypy
-    #   jaraco.classes
-    #   jaraco.functools
-moto==3.0.1 ; python_version >= "3.6"
-    # via -r requirements/static/ci/common.in
-msgpack==1.0.2
-    # via
-    #   -r requirements/base.txt
-    #   pytest-salt-factories
-multidict==6.0.2
-    # via
-    #   aiohttp
-    #   yarl
-napalm==3.1.0 ; sys_platform != "win32" and python_version > "3.6" and python_version < "3.10"
-    # via -r requirements/static/ci/common.in
-ncclient==0.6.4
-    # via junos-eznc
-netaddr==0.7.19
-    # via
-    #   junos-eznc
-    #   napalm
-    #   pyeapi
-netmiko==3.2.0
-    # via napalm
-ntc-templates==1.4.0
-    # via junos-eznc
-oscrypto==1.2.0
-    # via certvalidator
-packaging==21.3
-    # via
-    #   -r requirements/base.txt
-    #   pytest
-paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   junos-eznc
-    #   napalm
-    #   ncclient
-    #   netmiko
-    #   scp
-passlib[bcrypt]==1.7.4
-    # via
-    #   -r requirements/static/ci/common.in
-    #   ciscoconfparse
-pathspec==0.9.0
-    # via yamllint
-pathtools==0.1.2
-    # via watchdog
-platformdirs==2.2.0
-    # via virtualenv
-pluggy==0.13.0
-    # via pytest
-portend==2.4
-    # via cherrypy
-psutil==5.8.0
-    # via
-    #   -r requirements/base.txt
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-system-statistics
-pyasn1-modules==0.2.4
-    # via google-auth
-pyasn1==0.4.8
-    # via
-    #   pyasn1-modules
-    #   rsa
-pycparser==2.17
-    # via cffi
-pycryptodomex==3.9.8
-    # via -r requirements/crypto.txt
-pyeapi==0.8.3
-    # via napalm
-pygit2==1.8.0 ; python_version >= "3.7"
-    # via -r requirements/static/ci/freebsd.in
-pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd"
-    # via -r requirements/static/ci/common.in
-pynacl==1.3.0
-    # via paramiko
-pyopenssl==23.0.0
-    # via
-    #   -r requirements/static/pkg/freebsd.in
-    #   etcd3-py
-pyparsing==3.0.9
-    # via
-    #   junos-eznc
-    #   packaging
-pyrsistent==0.17.3
-    # via jsonschema
-pyserial==3.4
-    # via
-    #   junos-eznc
-    #   netmiko
-pytest-custom-exit-code==0.3.0
-    # via -r requirements/pytest.txt
-pytest-helpers-namespace==2021.4.29
-    # via
-    #   -r requirements/pytest.txt
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-pytest-httpserver==1.0.4
-    # via -r requirements/pytest.txt
-pytest-salt-factories[docker]==1.0.0rc21 ; sys_platform != "win32"
-    # via -r requirements/pytest.txt
-pytest-shell-utilities==1.6.0
-    # via pytest-salt-factories
-pytest-skip-markers==1.2.0
-    # via
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-system-statistics
-pytest-subtests==0.4.0
-    # via -r requirements/pytest.txt
-pytest-system-statistics==1.0.2
-    # via pytest-salt-factories
-pytest-tempdir==2019.10.12
-    # via
-    #   -r requirements/pytest.txt
-    #   pytest-salt-factories
-pytest-timeout==1.4.2
-    # via -r requirements/pytest.txt
-pytest==7.2.0 ; python_version > "3.6"
-    # via
-    #   -r requirements/pytest.txt
-    #   pytest-custom-exit-code
-    #   pytest-helpers-namespace
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-skip-markers
-    #   pytest-subtests
-    #   pytest-system-statistics
-    #   pytest-tempdir
-    #   pytest-timeout
-python-dateutil==2.8.1
-    # via
-    #   -r requirements/static/pkg/freebsd.in
-    #   botocore
-    #   croniter
-    #   kubernetes
-    #   moto
-    #   vcert
-python-etcd==0.4.5
-    # via -r requirements/static/ci/common.in
-python-gnupg==0.4.8
-    # via -r requirements/static/pkg/freebsd.in
-pytz==2022.1
-    # via
-    #   moto
-    #   tempora
-pyvmomi==6.7.1.2018.12
-    # via -r requirements/static/ci/common.in
-pyyaml==5.4.1
-    # via
-    #   -r requirements/base.txt
-    #   clustershell
-    #   junos-eznc
-    #   kubernetes
-    #   napalm
-    #   yamllint
-    #   yamlordereddictloader
-pyzmq==23.2.0 ; python_version < "3.11"
-    # via
-    #   -r requirements/zeromq.txt
-    #   pytest-salt-factories
-requests==2.31.0
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/static/ci/common.in
-    #   apache-libcloud
-    #   docker
-    #   etcd3-py
-    #   kubernetes
-    #   moto
-    #   napalm
-    #   pyvmomi
-    #   responses
-    #   vcert
-responses==0.10.6
-    # via moto
-rfc3987==1.3.8
-    # via -r requirements/static/ci/common.in
-rsa==4.7.2
-    # via google-auth
-s3transfer==0.5.2
-    # via boto3
-scp==0.13.2
-    # via
-    #   junos-eznc
-    #   napalm
-    #   netmiko
-semantic-version==2.9.0
-    # via etcd3-py
-setproctitle==1.3.2
-    # via -r requirements/static/pkg/freebsd.in
-six==1.16.0
-    # via
-    #   bcrypt
-    #   cassandra-driver
-    #   cheroot
-    #   etcd3-py
-    #   genshi
-    #   geomet
-    #   google-auth
-    #   jsonschema
-    #   junos-eznc
-    #   kazoo
-    #   kubernetes
-    #   mock
-    #   more-itertools
-    #   ncclient
-    #   paramiko
-    #   pynacl
-    #   python-dateutil
-    #   pyvmomi
-    #   responses
-    #   textfsm
-    #   transitions
-    #   vcert
-    #   virtualenv
-    #   websocket-client
-smmap==3.0.4
-    # via gitdb
-sqlparse==0.4.4
-    # via -r requirements/static/ci/common.in
-strict-rfc3339==0.7
-    # via -r requirements/static/ci/common.in
-tempora==4.1.1
-    # via portend
-terminal==0.4.0
-    # via ntc-templates
-textfsm==1.1.0
-    # via
-    #   napalm
-    #   netmiko
-    #   ntc-templates
-timelib==0.2.5
-    # via -r requirements/static/pkg/freebsd.in
-toml==0.10.2
-    # via -r requirements/static/ci/common.in
-tomli==2.0.1
-    # via pytest
-tornado==6.2.0 ; python_version < "3.8"
-    # via -r requirements/base.txt
-transitions==0.8.1
-    # via junos-eznc
-typing-extensions==3.10.0.0
-    # via
-    #   aiohttp
-    #   async-timeout
-    #   gitpython
-    #   importlib-metadata
-    #   pytest-shell-utilities
-    #   pytest-system-statistics
-    #   yarl
-urllib3==1.26.6
-    # via
-    #   botocore
-    #   kubernetes
-    #   python-etcd
-    #   requests
-vcert==0.7.4 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-virtualenv==20.7.2
-    # via
-    #   -r requirements/static/ci/common.in
-    #   pytest-salt-factories
-watchdog==0.10.3
-    # via -r requirements/static/ci/common.in
-websocket-client==0.40.0
-    # via
-    #   docker
-    #   kubernetes
-werkzeug==2.2.3
-    # via
-    #   moto
-    #   pytest-httpserver
-xmltodict==0.12.0
-    # via moto
-yamllint==1.26.3
-    # via -r requirements/static/ci/freebsd.in
-yamlordereddictloader==0.4.0
-    # via junos-eznc
-yarl==1.7.2
-    # via aiohttp
-zc.lockfile==1.4
-    # via cherrypy
-zipp==3.5.0
-    # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/requirements/static/ci/py3.7/invoke.txt b/requirements/static/ci/py3.7/invoke.txt
deleted file mode 100644
index e2cad5c72e0..00000000000
--- a/requirements/static/ci/py3.7/invoke.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/ci/py3.7/invoke.txt --pip-args='--constraint=requirements/static/ci/py3.7/linux.txt' requirements/static/ci/invoke.in
-#
-blessings==1.7
-    # via -r requirements/static/ci/invoke.in
-invoke==1.4.1
-    # via -r requirements/static/ci/invoke.in
-pyyaml==5.4.1
-    # via -r requirements/static/ci/invoke.in
-six==1.16.0
-    # via blessings
diff --git a/requirements/static/ci/py3.7/lint.txt b/requirements/static/ci/py3.7/lint.txt
deleted file mode 100644
index ffa5e99ea77..00000000000
--- a/requirements/static/ci/py3.7/lint.txt
+++ /dev/null
@@ -1,513 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/ci/py3.7/lint.txt --pip-args='--constraint=requirements/static/ci/py3.7/linux.txt' requirements/base.txt requirements/static/ci/common.in requirements/static/ci/lint.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt
-#
-aiohttp==3.8.1
-    # via etcd3-py
-aiosignal==1.2.0
-    # via aiohttp
-ansible-core==2.11.4
-    # via ansible
-ansible==4.4.0 ; python_version < "3.9"
-    # via -r requirements/static/ci/linux.in
-apache-libcloud==3.3.1 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-apscheduler==3.6.3
-    # via python-telegram-bot
-asn1crypto==1.4.0
-    # via
-    #   certvalidator
-    #   oscrypto
-astroid==2.3.3
-    # via pylint
-async-timeout==4.0.2
-    # via aiohttp
-asynctest==0.13.0
-    # via aiohttp
-attrs==21.2.0
-    # via
-    #   aiohttp
-    #   jsonschema
-backports.entry-points-selectable==1.1.0
-    # via virtualenv
-backports.zoneinfo==0.2.1
-    # via tzlocal
-bcrypt==3.2.0
-    # via
-    #   paramiko
-    #   passlib
-boto3==1.21.46 ; python_version >= "3.6"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   moto
-boto==2.49.0
-    # via -r requirements/static/ci/common.in
-botocore==1.24.46
-    # via
-    #   boto3
-    #   moto
-    #   s3transfer
-cachetools==4.2.2
-    # via
-    #   google-auth
-    #   python-telegram-bot
-cassandra-driver==3.25.0
-    # via -r requirements/static/ci/common.in
-certifi==2022.12.7
-    # via
-    #   -r requirements/static/ci/common.in
-    #   kubernetes
-    #   python-telegram-bot
-    #   requests
-certvalidator==0.11.1
-    # via vcert
-cffi==1.14.6
-    # via
-    #   -r requirements/static/ci/common.in
-    #   bcrypt
-    #   cryptography
-    #   napalm
-    #   pygit2
-    #   pynacl
-charset-normalizer==2.0.4
-    # via
-    #   aiohttp
-    #   requests
-cheetah3==3.2.6.post1
-    # via -r requirements/static/ci/common.in
-cheroot==8.5.2
-    # via cherrypy
-cherrypy==18.6.1
-    # via
-    #   -r requirements/static/ci/common.in
-    #   -r requirements/static/pkg/linux.in
-ciscoconfparse==1.5.46
-    # via napalm
-click==8.0.1
-    # via geomet
-clustershell==1.8.3
-    # via -r requirements/static/ci/common.in
-colorama==0.4.4
-    # via ciscoconfparse
-contextvars==2.4
-    # via -r requirements/base.txt
-croniter==1.0.15 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-cryptography==39.0.2
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   ansible-core
-    #   etcd3-py
-    #   moto
-    #   paramiko
-    #   pyopenssl
-    #   vcert
-distlib==0.3.2
-    # via virtualenv
-distro==1.6.0
-    # via -r requirements/base.txt
-dnspython==2.1.0
-    # via
-    #   -r requirements/static/ci/common.in
-    #   ciscoconfparse
-    #   python-etcd
-docker==5.0.0
-    # via -r requirements/static/ci/common.in
-etcd3-py==0.1.6 ; python_version >= "3.6"
-    # via -r requirements/static/ci/common.in
-filelock==3.0.12
-    # via virtualenv
-frozenlist==1.3.0
-    # via
-    #   aiohttp
-    #   aiosignal
-future==0.18.3
-    # via
-    #   napalm
-    #   textfsm
-genshi==0.7.5
-    # via -r requirements/static/ci/common.in
-geomet==0.2.1.post1
-    # via cassandra-driver
-gitdb==4.0.7
-    # via gitpython
-gitpython==3.1.30 ; python_version >= "3.7"
-    # via -r requirements/static/ci/common.in
-google-auth==2.0.1
-    # via kubernetes
-hglib==2.6.1
-    # via -r requirements/static/ci/linux.in
-idna==3.2
-    # via
-    #   etcd3-py
-    #   requests
-    #   yarl
-immutables==0.16
-    # via contextvars
-importlib-metadata==4.6.4
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   backports.entry-points-selectable
-    #   click
-    #   jsonschema
-    #   mako
-    #   moto
-    #   virtualenv
-ipaddress==1.0.23
-    # via kubernetes
-isort==4.3.21
-    # via pylint
-jaraco.classes==3.2.1
-    # via jaraco.collections
-jaraco.collections==3.4.0
-    # via cherrypy
-jaraco.functools==3.3.0
-    # via
-    #   cheroot
-    #   jaraco.text
-    #   tempora
-jaraco.text==3.5.1
-    # via jaraco.collections
-jinja2==3.1.2
-    # via
-    #   -r requirements/base.txt
-    #   ansible-core
-    #   junos-eznc
-    #   moto
-    #   napalm
-jmespath==0.10.0
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/static/ci/common.in
-    #   boto3
-    #   botocore
-jsonschema==3.2.0
-    # via -r requirements/static/ci/common.in
-junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   napalm
-jxmlease==1.0.3 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-kazoo==2.8.0 ; sys_platform != "win32" and sys_platform != "darwin"
-    # via -r requirements/static/ci/common.in
-keyring==5.7.1
-    # via -r requirements/static/ci/common.in
-kubernetes==3.0.0
-    # via -r requirements/static/ci/common.in
-lazy-object-proxy==1.4.3
-    # via astroid
-libnacl==1.8.0 ; sys_platform != "win32" and sys_platform != "darwin"
-    # via -r requirements/static/ci/common.in
-loguru==0.6.0
-    # via ciscoconfparse
-looseversion==1.0.2
-    # via -r requirements/base.txt
-lxml==4.9.1
-    # via
-    #   junos-eznc
-    #   napalm
-    #   ncclient
-mako==1.2.2
-    # via -r requirements/static/ci/common.in
-markupsafe==2.1.2
-    # via
-    #   -r requirements/base.txt
-    #   jinja2
-    #   mako
-    #   moto
-    #   werkzeug
-mccabe==0.6.1
-    # via pylint
-mercurial==6.0.1
-    # via -r requirements/static/ci/linux.in
-modernize==0.5
-    # via saltpylint
-more-itertools==8.8.0
-    # via
-    #   cheroot
-    #   cherrypy
-    #   jaraco.classes
-    #   jaraco.functools
-moto==3.0.1 ; python_version >= "3.6"
-    # via -r requirements/static/ci/common.in
-msgpack==1.0.2
-    # via -r requirements/base.txt
-multidict==6.0.2
-    # via
-    #   aiohttp
-    #   yarl
-napalm==3.3.1 ; sys_platform != "win32" and python_version > "3.6" and python_version < "3.10"
-    # via -r requirements/static/ci/common.in
-ncclient==0.6.12
-    # via
-    #   junos-eznc
-    #   napalm
-netaddr==0.8.0
-    # via
-    #   junos-eznc
-    #   napalm
-    #   pyeapi
-netmiko==3.4.0
-    # via napalm
-ntc-templates==2.2.2
-    # via
-    #   junos-eznc
-    #   netmiko
-oscrypto==1.2.1
-    # via certvalidator
-packaging==21.3
-    # via
-    #   -r requirements/base.txt
-    #   ansible-core
-paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   junos-eznc
-    #   napalm
-    #   ncclient
-    #   netmiko
-    #   scp
-passlib[bcrypt]==1.7.4
-    # via
-    #   -r requirements/static/ci/common.in
-    #   ciscoconfparse
-pathspec==0.9.0
-    # via yamllint
-platformdirs==2.2.0
-    # via virtualenv
-portend==2.7.1
-    # via cherrypy
-psutil==5.8.0
-    # via -r requirements/base.txt
-pyasn1-modules==0.2.8
-    # via google-auth
-pyasn1==0.4.8
-    # via
-    #   pyasn1-modules
-    #   rsa
-pycodestyle==2.5.0
-    # via saltpylint
-pycparser==2.20
-    # via cffi
-pycryptodomex==3.10.1
-    # via -r requirements/crypto.txt
-pyeapi==0.8.4
-    # via napalm
-pygit2==1.0.3 ; python_version <= "3.8"
-    # via -r requirements/static/ci/linux.in
-pyiface==0.0.11
-    # via -r requirements/static/ci/linux.in
-pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd"
-    # via -r requirements/static/ci/common.in
-pyjwt==2.4.0
-    # via twilio
-pylint==2.4.4
-    # via
-    #   -r requirements/static/ci/lint.in
-    #   saltpylint
-pymysql==1.0.2 ; python_version > "3.5"
-    # via -r requirements/static/ci/linux.in
-pynacl==1.4.0
-    # via paramiko
-pyopenssl==23.0.0
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   etcd3-py
-pyparsing==3.0.9
-    # via
-    #   junos-eznc
-    #   packaging
-pyrsistent==0.18.0
-    # via jsonschema
-pyserial==3.5
-    # via
-    #   junos-eznc
-    #   netmiko
-python-consul==1.1.0
-    # via -r requirements/static/ci/linux.in
-python-dateutil==2.8.2
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   botocore
-    #   croniter
-    #   kubernetes
-    #   moto
-    #   vcert
-python-etcd==0.4.5
-    # via -r requirements/static/ci/common.in
-python-gnupg==0.4.8
-    # via -r requirements/static/pkg/linux.in
-python-telegram-bot==13.7 ; python_version > "3.5"
-    # via -r requirements/static/ci/linux.in
-pytz==2022.1
-    # via
-    #   apscheduler
-    #   moto
-    #   python-telegram-bot
-    #   tempora
-    #   twilio
-pyvmomi==7.0.2
-    # via -r requirements/static/ci/common.in
-pyyaml==5.4.1
-    # via
-    #   -r requirements/base.txt
-    #   ansible-core
-    #   clustershell
-    #   junos-eznc
-    #   kubernetes
-    #   napalm
-    #   yamllint
-    #   yamlordereddictloader
-pyzmq==23.2.0 ; python_version < "3.11"
-    # via -r requirements/zeromq.txt
-redis-py-cluster==2.1.3
-    # via -r requirements/static/ci/linux.in
-redis==3.5.3
-    # via redis-py-cluster
-requests==2.31.0
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/static/ci/common.in
-    #   apache-libcloud
-    #   docker
-    #   etcd3-py
-    #   kubernetes
-    #   moto
-    #   napalm
-    #   python-consul
-    #   pyvmomi
-    #   responses
-    #   twilio
-    #   vcert
-resolvelib==0.5.4
-    # via ansible-core
-responses==0.13.4
-    # via moto
-rfc3987==1.3.8
-    # via -r requirements/static/ci/common.in
-rpm-vercmp==0.1.2
-    # via -r requirements/static/pkg/linux.in
-rsa==4.7.2
-    # via google-auth
-s3transfer==0.5.0
-    # via boto3
-saltpylint==2020.9.28
-    # via -r requirements/static/ci/lint.in
-scp==0.13.6
-    # via
-    #   junos-eznc
-    #   napalm
-    #   netmiko
-semantic-version==2.9.0
-    # via etcd3-py
-setproctitle==1.3.2
-    # via -r requirements/static/pkg/linux.in
-six==1.16.0
-    # via
-    #   apscheduler
-    #   astroid
-    #   bcrypt
-    #   cassandra-driver
-    #   cheroot
-    #   etcd3-py
-    #   genshi
-    #   geomet
-    #   jsonschema
-    #   junos-eznc
-    #   kazoo
-    #   kubernetes
-    #   ncclient
-    #   paramiko
-    #   pynacl
-    #   python-consul
-    #   python-dateutil
-    #   pyvmomi
-    #   responses
-    #   textfsm
-    #   transitions
-    #   vcert
-    #   virtualenv
-    #   websocket-client
-slack-bolt==1.15.5
-    # via -r requirements/static/ci/linux.in
-slack-sdk==3.19.5
-    # via slack-bolt
-smmap==4.0.0
-    # via gitdb
-sqlparse==0.4.4
-    # via -r requirements/static/ci/common.in
-strict-rfc3339==0.7
-    # via -r requirements/static/ci/common.in
-tempora==4.1.1
-    # via portend
-tenacity==8.0.1
-    # via netmiko
-textfsm==1.1.2
-    # via
-    #   napalm
-    #   ntc-templates
-timelib==0.2.5
-    # via -r requirements/static/pkg/linux.in
-toml==0.10.2
-    # via
-    #   -r requirements/static/ci/common.in
-    #   -r requirements/static/ci/lint.in
-tornado==6.2 ; python_version < "3.8"
-    # via
-    #   -r requirements/base.txt
-    #   python-telegram-bot
-transitions==0.8.8
-    # via junos-eznc
-twilio==7.9.2
-    # via -r requirements/static/ci/linux.in
-typed-ast==1.4.1
-    # via astroid
-typing-extensions==3.10.0.0
-    # via
-    #   aiohttp
-    #   async-timeout
-    #   gitpython
-    #   immutables
-    #   importlib-metadata
-    #   yarl
-tzlocal==3.0
-    # via apscheduler
-urllib3==1.26.6
-    # via
-    #   botocore
-    #   kubernetes
-    #   python-etcd
-    #   requests
-    #   responses
-vcert==0.7.4 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-virtualenv==20.7.2
-    # via -r requirements/static/ci/common.in
-watchdog==2.1.5
-    # via -r requirements/static/ci/common.in
-websocket-client==0.40.0
-    # via
-    #   docker
-    #   kubernetes
-werkzeug==2.2.3
-    # via moto
-wrapt==1.11.1
-    # via astroid
-xmltodict==0.12.0
-    # via moto
-yamllint==1.26.3
-    # via -r requirements/static/ci/linux.in
-yamlordereddictloader==0.4.0
-    # via junos-eznc
-yarl==1.7.2
-    # via aiohttp
-zc.lockfile==2.0
-    # via cherrypy
-zipp==3.5.0
-    # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/requirements/static/ci/py3.7/linux-crypto.txt b/requirements/static/ci/py3.7/linux-crypto.txt
deleted file mode 100644
index 1c23cef2513..00000000000
--- a/requirements/static/ci/py3.7/linux-crypto.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/ci/py3.7/linux-crypto.txt --pip-args='--constraint=requirements/static/pkg/py3.7/linux.txt' requirements/static/ci/crypto.in
-#
-m2crypto==0.38.0
-    # via -r requirements/static/ci/crypto.in
-pycryptodome==3.9.7
-    # via -r requirements/static/ci/crypto.in
diff --git a/requirements/static/ci/py3.7/linux.txt b/requirements/static/ci/py3.7/linux.txt
deleted file mode 100644
index 8cf97a0956b..00000000000
--- a/requirements/static/ci/py3.7/linux.txt
+++ /dev/null
@@ -1,562 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/ci/py3.7/linux.txt --pip-args='--constraint=requirements/static/pkg/py3.7/linux.txt' requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/linux.in requirements/static/pkg/linux.in requirements/zeromq.txt
-#
-aiohttp==3.8.1
-    # via etcd3-py
-aiosignal==1.2.0
-    # via aiohttp
-ansible-core==2.11.7
-    # via ansible
-ansible==4.4.0 ; python_version < "3.9"
-    # via -r requirements/static/ci/linux.in
-apache-libcloud==2.5.0 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-apscheduler==3.6.3
-    # via python-telegram-bot
-asn1crypto==1.3.0
-    # via
-    #   certvalidator
-    #   oscrypto
-async-timeout==4.0.2
-    # via aiohttp
-asynctest==0.13.0
-    # via aiohttp
-attrs==20.3.0
-    # via
-    #   aiohttp
-    #   jsonschema
-    #   pytest
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-skip-markers
-    #   pytest-system-statistics
-backports.entry-points-selectable==1.1.0
-    # via virtualenv
-bcrypt==3.1.6
-    # via
-    #   paramiko
-    #   passlib
-boto3==1.21.46 ; python_version >= "3.6"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   moto
-boto==2.49.0
-    # via -r requirements/static/ci/common.in
-botocore==1.24.46
-    # via
-    #   boto3
-    #   moto
-    #   s3transfer
-cachetools==4.2.2
-    # via
-    #   google-auth
-    #   python-telegram-bot
-cassandra-driver==3.23.0
-    # via -r requirements/static/ci/common.in
-certifi==2022.12.7
-    # via
-    #   -r requirements/static/ci/common.in
-    #   kubernetes
-    #   python-telegram-bot
-    #   requests
-certvalidator==0.11.1
-    # via vcert
-cffi==1.14.6
-    # via
-    #   -r requirements/static/ci/common.in
-    #   bcrypt
-    #   cryptography
-    #   napalm
-    #   pygit2
-    #   pynacl
-charset-normalizer==2.0.12
-    # via
-    #   aiohttp
-    #   requests
-cheetah3==3.2.6.post2
-    # via -r requirements/static/ci/common.in
-cheroot==8.5.2
-    # via cherrypy
-cherrypy==18.6.1
-    # via
-    #   -r requirements/static/ci/common.in
-    #   -r requirements/static/pkg/linux.in
-ciscoconfparse==1.5.19
-    # via napalm
-click==7.1.1
-    # via geomet
-clustershell==1.8.3
-    # via -r requirements/static/ci/common.in
-colorama==0.4.3
-    # via ciscoconfparse
-contextvars==2.4
-    # via -r requirements/base.txt
-croniter==0.3.29 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-cryptography==39.0.2
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   ansible-core
-    #   etcd3-py
-    #   moto
-    #   paramiko
-    #   pyopenssl
-    #   vcert
-distlib==0.3.2
-    # via virtualenv
-distro==1.5.0
-    # via
-    #   -r requirements/base.txt
-    #   pytest-skip-markers
-dnspython==1.16.0
-    # via
-    #   -r requirements/static/ci/common.in
-    #   ciscoconfparse
-    #   python-etcd
-docker==5.0.3
-    # via
-    #   -r requirements/static/ci/common.in
-    #   pytest-salt-factories
-etcd3-py==0.1.6 ; python_version >= "3.6"
-    # via -r requirements/static/ci/common.in
-exceptiongroup==1.0.4
-    # via pytest
-filelock==3.0.12
-    # via virtualenv
-flaky==3.7.0
-    # via -r requirements/pytest.txt
-frozenlist==1.3.0
-    # via
-    #   aiohttp
-    #   aiosignal
-future==0.18.3
-    # via
-    #   napalm
-    #   textfsm
-genshi==0.7.5
-    # via -r requirements/static/ci/common.in
-geomet==0.1.2
-    # via cassandra-driver
-gitdb==4.0.5
-    # via gitpython
-gitpython==3.1.30 ; python_version >= "3.7"
-    # via -r requirements/static/ci/common.in
-google-auth==1.6.3
-    # via kubernetes
-hglib==2.6.1
-    # via -r requirements/static/ci/linux.in
-idna==2.8
-    # via
-    #   etcd3-py
-    #   requests
-    #   yarl
-immutables==0.15
-    # via contextvars
-importlib-metadata==4.6.4
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   backports.entry-points-selectable
-    #   jsonschema
-    #   mako
-    #   moto
-    #   pluggy
-    #   pytest
-    #   virtualenv
-iniconfig==1.0.1
-    # via pytest
-ipaddress==1.0.22
-    # via kubernetes
-jaraco.classes==3.2.1
-    # via jaraco.collections
-jaraco.collections==3.4.0
-    # via cherrypy
-jaraco.functools==2.0
-    # via
-    #   cheroot
-    #   jaraco.text
-    #   tempora
-jaraco.text==3.5.1
-    # via jaraco.collections
-jinja2==3.1.2
-    # via
-    #   -r requirements/base.txt
-    #   ansible-core
-    #   junos-eznc
-    #   moto
-    #   napalm
-jmespath==1.0.1
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/static/ci/common.in
-    #   boto3
-    #   botocore
-jsonschema==3.2.0
-    # via -r requirements/static/ci/common.in
-junos-eznc==2.4.0 ; sys_platform != "win32" and python_version <= "3.10"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   napalm
-jxmlease==1.0.1 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-kazoo==2.6.1 ; sys_platform != "win32" and sys_platform != "darwin"
-    # via -r requirements/static/ci/common.in
-keyring==5.7.1
-    # via -r requirements/static/ci/common.in
-kubernetes==3.0.0
-    # via -r requirements/static/ci/common.in
-libnacl==1.7.1 ; sys_platform != "win32" and sys_platform != "darwin"
-    # via -r requirements/static/ci/common.in
-looseversion==1.0.2
-    # via -r requirements/base.txt
-lxml==4.9.1
-    # via
-    #   junos-eznc
-    #   napalm
-    #   ncclient
-mako==1.2.2
-    # via -r requirements/static/ci/common.in
-markupsafe==2.1.2
-    # via
-    #   -r requirements/base.txt
-    #   jinja2
-    #   mako
-    #   moto
-    #   werkzeug
-mercurial==6.0.1
-    # via -r requirements/static/ci/linux.in
-mock==3.0.5
-    # via -r requirements/pytest.txt
-more-itertools==5.0.0
-    # via
-    #   cheroot
-    #   cherrypy
-    #   jaraco.classes
-    #   jaraco.functools
-moto==3.0.1 ; python_version >= "3.6"
-    # via -r requirements/static/ci/common.in
-msgpack==1.0.2
-    # via
-    #   -r requirements/base.txt
-    #   pytest-salt-factories
-multidict==6.0.2
-    # via
-    #   aiohttp
-    #   yarl
-napalm==3.1.0 ; sys_platform != "win32" and python_version > "3.6" and python_version < "3.10"
-    # via -r requirements/static/ci/common.in
-ncclient==0.6.4
-    # via junos-eznc
-netaddr==0.7.19
-    # via
-    #   junos-eznc
-    #   napalm
-    #   pyeapi
-netmiko==3.2.0
-    # via napalm
-ntc-templates==1.4.0
-    # via junos-eznc
-oscrypto==1.2.0
-    # via certvalidator
-packaging==21.3
-    # via
-    #   -r requirements/base.txt
-    #   ansible-core
-    #   pytest
-paramiko==2.10.1 ; sys_platform != "win32" and sys_platform != "darwin"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   junos-eznc
-    #   napalm
-    #   ncclient
-    #   netmiko
-    #   scp
-passlib[bcrypt]==1.7.4
-    # via
-    #   -r requirements/static/ci/common.in
-    #   ciscoconfparse
-pathspec==0.9.0
-    # via yamllint
-pathtools==0.1.2
-    # via watchdog
-platformdirs==2.2.0
-    # via virtualenv
-pluggy==0.13.0
-    # via pytest
-portend==2.4
-    # via cherrypy
-psutil==5.8.0
-    # via
-    #   -r requirements/base.txt
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-system-statistics
-pyasn1-modules==0.2.4
-    # via google-auth
-pyasn1==0.4.8
-    # via
-    #   pyasn1-modules
-    #   rsa
-pycparser==2.17
-    # via cffi
-pycryptodomex==3.9.8
-    # via -r requirements/crypto.txt
-pyeapi==0.8.3
-    # via napalm
-pygit2==1.0.3 ; python_version <= "3.8"
-    # via -r requirements/static/ci/linux.in
-pyiface==0.0.11
-    # via -r requirements/static/ci/linux.in
-pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd"
-    # via -r requirements/static/ci/common.in
-pyjwt==2.4.0
-    # via twilio
-pymysql==1.0.2 ; python_version > "3.5"
-    # via -r requirements/static/ci/linux.in
-pynacl==1.3.0
-    # via paramiko
-pyopenssl==23.0.0
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   etcd3-py
-pyparsing==3.0.9
-    # via
-    #   junos-eznc
-    #   packaging
-pyrsistent==0.17.3
-    # via jsonschema
-pyserial==3.4
-    # via
-    #   junos-eznc
-    #   netmiko
-pytest-custom-exit-code==0.3.0
-    # via -r requirements/pytest.txt
-pytest-helpers-namespace==2021.4.29
-    # via
-    #   -r requirements/pytest.txt
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-pytest-httpserver==1.0.4
-    # via -r requirements/pytest.txt
-pytest-salt-factories[docker]==1.0.0rc21 ; sys_platform != "win32"
-    # via -r requirements/pytest.txt
-pytest-shell-utilities==1.6.0
-    # via pytest-salt-factories
-pytest-skip-markers==1.2.0
-    # via
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-system-statistics
-pytest-subtests==0.4.0
-    # via -r requirements/pytest.txt
-pytest-system-statistics==1.0.2
-    # via pytest-salt-factories
-pytest-tempdir==2019.10.12
-    # via
-    #   -r requirements/pytest.txt
-    #   pytest-salt-factories
-pytest-timeout==1.4.2
-    # via -r requirements/pytest.txt
-pytest==7.2.0 ; python_version > "3.6"
-    # via
-    #   -r requirements/pytest.txt
-    #   pytest-custom-exit-code
-    #   pytest-helpers-namespace
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-skip-markers
-    #   pytest-subtests
-    #   pytest-system-statistics
-    #   pytest-tempdir
-    #   pytest-timeout
-python-consul==1.1.0
-    # via -r requirements/static/ci/linux.in
-python-dateutil==2.8.1
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   botocore
-    #   croniter
-    #   kubernetes
-    #   moto
-    #   vcert
-python-etcd==0.4.5
-    # via -r requirements/static/ci/common.in
-python-gnupg==0.4.8
-    # via -r requirements/static/pkg/linux.in
-python-telegram-bot==13.7 ; python_version > "3.5"
-    # via -r requirements/static/ci/linux.in
-pytz==2022.1
-    # via
-    #   apscheduler
-    #   moto
-    #   python-telegram-bot
-    #   tempora
-    #   twilio
-    #   tzlocal
-pyvmomi==6.7.1.2018.12
-    # via -r requirements/static/ci/common.in
-pyyaml==5.4.1
-    # via
-    #   -r requirements/base.txt
-    #   ansible-core
-    #   clustershell
-    #   junos-eznc
-    #   kubernetes
-    #   napalm
-    #   yamllint
-    #   yamlordereddictloader
-pyzmq==23.2.0 ; python_version < "3.11"
-    # via
-    #   -r requirements/zeromq.txt
-    #   pytest-salt-factories
-redis-py-cluster==2.1.3
-    # via -r requirements/static/ci/linux.in
-redis==3.5.3
-    # via redis-py-cluster
-requests==2.31.0
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/static/ci/common.in
-    #   apache-libcloud
-    #   docker
-    #   etcd3-py
-    #   kubernetes
-    #   moto
-    #   napalm
-    #   python-consul
-    #   pyvmomi
-    #   responses
-    #   twilio
-    #   vcert
-resolvelib==0.5.4
-    # via ansible-core
-responses==0.10.6
-    # via moto
-rfc3987==1.3.8
-    # via -r requirements/static/ci/common.in
-rpm-vercmp==0.1.2
-    # via -r requirements/static/pkg/linux.in
-rsa==4.7.2
-    # via google-auth
-s3transfer==0.5.2
-    # via boto3
-scp==0.13.2
-    # via
-    #   junos-eznc
-    #   napalm
-    #   netmiko
-semantic-version==2.9.0
-    # via etcd3-py
-setproctitle==1.3.2
-    # via -r requirements/static/pkg/linux.in
-six==1.16.0
-    # via
-    #   apscheduler
-    #   bcrypt
-    #   cassandra-driver
-    #   cheroot
-    #   etcd3-py
-    #   genshi
-    #   geomet
-    #   google-auth
-    #   jsonschema
-    #   junos-eznc
-    #   kazoo
-    #   kubernetes
-    #   mock
-    #   more-itertools
-    #   ncclient
-    #   paramiko
-    #   pynacl
-    #   python-consul
-    #   python-dateutil
-    #   pyvmomi
-    #   responses
-    #   textfsm
-    #   transitions
-    #   vcert
-    #   virtualenv
-    #   websocket-client
-slack-bolt==1.15.5
-    # via -r requirements/static/ci/linux.in
-slack-sdk==3.19.5
-    # via slack-bolt
-smmap==3.0.4
-    # via gitdb
-sqlparse==0.4.4
-    # via -r requirements/static/ci/common.in
-strict-rfc3339==0.7
-    # via -r requirements/static/ci/common.in
-tempora==4.1.1
-    # via portend
-terminal==0.4.0
-    # via ntc-templates
-textfsm==1.1.0
-    # via
-    #   napalm
-    #   netmiko
-    #   ntc-templates
-timelib==0.2.5
-    # via -r requirements/static/pkg/linux.in
-toml==0.10.2
-    # via -r requirements/static/ci/common.in
-tomli==2.0.1
-    # via pytest
-tornado==6.2 ; python_version < "3.8"
-    # via
-    #   -r requirements/base.txt
-    #   python-telegram-bot
-transitions==0.8.1
-    # via junos-eznc
-twilio==7.9.2
-    # via -r requirements/static/ci/linux.in
-typing-extensions==3.10.0.0
-    # via
-    #   aiohttp
-    #   async-timeout
-    #   gitpython
-    #   importlib-metadata
-    #   pytest-shell-utilities
-    #   pytest-system-statistics
-    #   yarl
-tzlocal==2.1
-    # via apscheduler
-urllib3==1.26.6
-    # via
-    #   botocore
-    #   kubernetes
-    #   python-etcd
-    #   requests
-vcert==0.7.4 ; sys_platform != "win32"
-    # via -r requirements/static/ci/common.in
-virtualenv==20.7.2
-    # via
-    #   -r requirements/static/ci/common.in
-    #   pytest-salt-factories
-watchdog==0.10.3
-    # via -r requirements/static/ci/common.in
-websocket-client==0.40.0
-    # via
-    #   docker
-    #   kubernetes
-werkzeug==2.2.3
-    # via
-    #   moto
-    #   pytest-httpserver
-xmltodict==0.12.0
-    # via moto
-yamllint==1.26.3
-    # via -r requirements/static/ci/linux.in
-yamlordereddictloader==0.4.0
-    # via junos-eznc
-yarl==1.7.2
-    # via aiohttp
-zc.lockfile==1.4
-    # via cherrypy
-zipp==3.5.0
-    # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/requirements/static/ci/py3.7/windows-crypto.txt b/requirements/static/ci/py3.7/windows-crypto.txt
deleted file mode 100644
index d3c32844914..00000000000
--- a/requirements/static/ci/py3.7/windows-crypto.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/ci/py3.7/windows-crypto.txt --pip-args='--constraint=requirements/static/ci/py3.7/windows.txt' requirements/static/ci/crypto.in
-#
-m2crypto==0.38.0
-    # via -r requirements/static/ci/crypto.in
-pycryptodome==3.9.7
-    # via -r requirements/static/ci/crypto.in
diff --git a/requirements/static/ci/py3.7/windows.txt b/requirements/static/ci/py3.7/windows.txt
deleted file mode 100644
index 6d62a314de9..00000000000
--- a/requirements/static/ci/py3.7/windows.txt
+++ /dev/null
@@ -1,449 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/ci/py3.7/windows.txt --pip-args='--constraint=requirements/static/pkg/py3.7/windows.txt' requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/windows.in requirements/static/pkg/windows.in requirements/windows.txt
-#
-aiohttp==3.8.1
-    # via etcd3-py
-aiosignal==1.2.0
-    # via aiohttp
-async-timeout==4.0.2
-    # via aiohttp
-asynctest==0.13.0
-    # via aiohttp
-attrs==20.3.0
-    # via
-    #   aiohttp
-    #   jsonschema
-    #   pytest
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-skip-markers
-    #   pytest-system-statistics
-backports.entry-points-selectable==1.1.0
-    # via virtualenv
-bcrypt==4.0.1
-    # via passlib
-boto3==1.21.46 ; python_version >= "3.6"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   moto
-boto==2.49.0
-    # via -r requirements/static/ci/common.in
-botocore==1.24.46
-    # via
-    #   boto3
-    #   moto
-    #   s3transfer
-cached-property==1.5.2
-    # via pygit2
-cachetools==3.1.0
-    # via google-auth
-cassandra-driver==3.23.0
-    # via -r requirements/static/ci/common.in
-certifi==2022.12.7
-    # via
-    #   -r requirements/static/ci/common.in
-    #   -r requirements/windows.txt
-    #   kubernetes
-    #   requests
-cffi==1.14.6
-    # via
-    #   -r requirements/static/ci/common.in
-    #   -r requirements/windows.txt
-    #   clr-loader
-    #   cryptography
-    #   pygit2
-charset-normalizer==2.0.12
-    # via
-    #   aiohttp
-    #   requests
-cheetah3==3.2.6.post2
-    # via -r requirements/static/ci/common.in
-cheroot==8.5.2
-    # via cherrypy
-cherrypy==18.6.1
-    # via
-    #   -r requirements/static/ci/common.in
-    #   -r requirements/windows.txt
-click==7.1.2
-    # via geomet
-clr-loader==0.2.4
-    # via pythonnet
-clustershell==1.8.3
-    # via -r requirements/static/ci/common.in
-colorama==0.4.1
-    # via pytest
-contextvars==2.4
-    # via -r requirements/base.txt
-cryptography==39.0.2
-    # via
-    #   -r requirements/windows.txt
-    #   etcd3-py
-    #   moto
-    #   pyopenssl
-    #   requests-ntlm
-distlib==0.3.2
-    # via virtualenv
-distro==1.5.0
-    # via
-    #   -r requirements/base.txt
-    #   pytest-skip-markers
-dmidecode==0.9.0
-    # via -r requirements/static/ci/windows.in
-dnspython==1.16.0
-    # via
-    #   -r requirements/static/ci/common.in
-    #   python-etcd
-docker-pycreds==0.4.0
-    # via docker
-docker==2.7.0
-    # via -r requirements/static/ci/common.in
-etcd3-py==0.1.6 ; python_version >= "3.6"
-    # via -r requirements/static/ci/common.in
-exceptiongroup==1.0.4
-    # via pytest
-filelock==3.0.12
-    # via virtualenv
-flaky==3.7.0
-    # via -r requirements/pytest.txt
-frozenlist==1.3.0
-    # via
-    #   aiohttp
-    #   aiosignal
-genshi==0.7.5
-    # via -r requirements/static/ci/common.in
-geomet==0.1.2
-    # via cassandra-driver
-gitdb==4.0.7
-    # via gitpython
-gitpython==3.1.30 ; python_version >= "3.7"
-    # via
-    #   -r requirements/static/ci/common.in
-    #   -r requirements/windows.txt
-google-auth==1.6.3
-    # via kubernetes
-idna==2.8
-    # via
-    #   etcd3-py
-    #   requests
-    #   yarl
-immutables==0.15
-    # via contextvars
-importlib-metadata==4.6.4
-    # via
-    #   -r requirements/windows.txt
-    #   backports.entry-points-selectable
-    #   jsonschema
-    #   mako
-    #   moto
-    #   pluggy
-    #   pytest
-    #   virtualenv
-iniconfig==1.0.1
-    # via pytest
-ioloop==0.1a0
-    # via -r requirements/windows.txt
-ipaddress==1.0.22
-    # via kubernetes
-jaraco.classes==3.2.1
-    # via jaraco.collections
-jaraco.collections==3.3.0
-    # via cherrypy
-jaraco.functools==2.0
-    # via
-    #   cheroot
-    #   jaraco.text
-    #   tempora
-jaraco.text==3.5.0
-    # via jaraco.collections
-jinja2==3.1.2
-    # via
-    #   -r requirements/base.txt
-    #   moto
-jmespath==1.0.1
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/static/ci/common.in
-    #   boto3
-    #   botocore
-jsonschema==3.2.0
-    # via -r requirements/static/ci/common.in
-keyring==5.7.1
-    # via -r requirements/static/ci/common.in
-kubernetes==3.0.0
-    # via -r requirements/static/ci/common.in
-looseversion==1.0.2
-    # via -r requirements/base.txt
-lxml==4.9.1
-    # via -r requirements/windows.txt
-mako==1.2.2
-    # via -r requirements/static/ci/common.in
-markupsafe==2.1.2
-    # via
-    #   -r requirements/base.txt
-    #   jinja2
-    #   mako
-    #   moto
-    #   werkzeug
-mock==3.0.5
-    # via -r requirements/pytest.txt
-more-itertools==8.2.0
-    # via
-    #   cheroot
-    #   cherrypy
-    #   jaraco.classes
-    #   jaraco.functools
-moto==3.0.1 ; python_version >= "3.6"
-    # via -r requirements/static/ci/common.in
-msgpack==1.0.2
-    # via
-    #   -r requirements/base.txt
-    #   pytest-salt-factories
-multidict==6.0.2
-    # via
-    #   aiohttp
-    #   yarl
-ntlm-auth==1.5.0
-    # via requests-ntlm
-packaging==21.3
-    # via
-    #   -r requirements/base.txt
-    #   pytest
-passlib[bcrypt]==1.7.4
-    # via -r requirements/static/ci/common.in
-patch==1.16
-    # via -r requirements/static/ci/windows.in
-pathspec==0.9.0
-    # via yamllint
-pathtools==0.1.2
-    # via watchdog
-platformdirs==2.2.0
-    # via virtualenv
-pluggy==0.13.0
-    # via pytest
-portend==2.6
-    # via cherrypy
-psutil==5.8.0
-    # via
-    #   -r requirements/base.txt
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-system-statistics
-pyasn1-modules==0.2.4
-    # via google-auth
-pyasn1==0.4.8
-    # via
-    #   -r requirements/windows.txt
-    #   pyasn1-modules
-    #   rsa
-pycparser==2.21
-    # via
-    #   -r requirements/windows.txt
-    #   cffi
-pycryptodomex==3.10.1
-    # via -r requirements/crypto.txt
-pygit2==1.9.1 ; python_version >= "3.7"
-    # via -r requirements/static/ci/windows.in
-pymssql==2.2.1
-    # via -r requirements/windows.txt
-pymysql==1.0.2
-    # via -r requirements/windows.txt
-pyopenssl==23.0.0
-    # via
-    #   -r requirements/windows.txt
-    #   etcd3-py
-pyparsing==3.0.9
-    # via packaging
-pyrsistent==0.17.3
-    # via jsonschema
-pytest-custom-exit-code==0.3.0
-    # via -r requirements/pytest.txt
-pytest-helpers-namespace==2021.4.29
-    # via
-    #   -r requirements/pytest.txt
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-pytest-httpserver==1.0.4
-    # via -r requirements/pytest.txt
-pytest-salt-factories==1.0.0rc21 ; sys_platform == "win32"
-    # via -r requirements/pytest.txt
-pytest-shell-utilities==1.6.0
-    # via pytest-salt-factories
-pytest-skip-markers==1.2.0
-    # via
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-system-statistics
-pytest-subtests==0.4.0
-    # via -r requirements/pytest.txt
-pytest-system-statistics==1.0.2
-    # via pytest-salt-factories
-pytest-tempdir==2019.10.12
-    # via
-    #   -r requirements/pytest.txt
-    #   pytest-salt-factories
-pytest-timeout==1.4.2
-    # via -r requirements/pytest.txt
-pytest==7.2.0 ; python_version > "3.6"
-    # via
-    #   -r requirements/pytest.txt
-    #   pytest-custom-exit-code
-    #   pytest-helpers-namespace
-    #   pytest-salt-factories
-    #   pytest-shell-utilities
-    #   pytest-skip-markers
-    #   pytest-subtests
-    #   pytest-system-statistics
-    #   pytest-tempdir
-    #   pytest-timeout
-python-dateutil==2.8.1
-    # via
-    #   -r requirements/windows.txt
-    #   botocore
-    #   kubernetes
-    #   moto
-python-etcd==0.4.5
-    # via -r requirements/static/ci/common.in
-python-gnupg==0.4.8
-    # via -r requirements/windows.txt
-pythonnet==3.0.1
-    # via -r requirements/windows.txt
-pytz==2022.1
-    # via
-    #   moto
-    #   tempora
-pyvmomi==6.7.1.2018.12
-    # via -r requirements/static/ci/common.in
-pywin32==305
-    # via
-    #   -r requirements/windows.txt
-    #   cherrypy
-    #   pytest-skip-markers
-    #   wmi
-pywinrm==0.4.1
-    # via -r requirements/static/ci/windows.in
-pyyaml==5.4.1
-    # via
-    #   -r requirements/base.txt
-    #   clustershell
-    #   kubernetes
-    #   yamllint
-pyzmq==25.0.2 ; sys_platform == "win32"
-    # via
-    #   -r requirements/zeromq.txt
-    #   pytest-salt-factories
-requests-ntlm==1.1.0
-    # via pywinrm
-requests==2.31.0
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/static/ci/common.in
-    #   -r requirements/windows.txt
-    #   docker
-    #   etcd3-py
-    #   kubernetes
-    #   moto
-    #   pyvmomi
-    #   pywinrm
-    #   requests-ntlm
-    #   responses
-responses==0.10.6
-    # via moto
-rfc3987==1.3.8
-    # via -r requirements/static/ci/common.in
-rsa==4.7.2
-    # via google-auth
-s3transfer==0.5.2
-    # via boto3
-sed==0.3.1
-    # via -r requirements/static/ci/windows.in
-semantic-version==2.9.0
-    # via etcd3-py
-setproctitle==1.3.2
-    # via -r requirements/windows.txt
-six==1.15.0
-    # via
-    #   cassandra-driver
-    #   cheroot
-    #   docker
-    #   docker-pycreds
-    #   etcd3-py
-    #   genshi
-    #   geomet
-    #   google-auth
-    #   jsonschema
-    #   kubernetes
-    #   mock
-    #   python-dateutil
-    #   pyvmomi
-    #   pywinrm
-    #   responses
-    #   virtualenv
-    #   websocket-client
-smmap==4.0.0
-    # via gitdb
-sqlparse==0.4.4
-    # via -r requirements/static/ci/common.in
-strict-rfc3339==0.7
-    # via -r requirements/static/ci/common.in
-tempora==4.1.1
-    # via portend
-timelib==0.2.5
-    # via -r requirements/windows.txt
-toml==0.10.2
-    # via -r requirements/static/ci/common.in
-tomli==2.0.1
-    # via pytest
-tornado==6.2.0 ; python_version < "3.8"
-    # via -r requirements/base.txt
-typing-extensions==4.2.0
-    # via
-    #   aiohttp
-    #   async-timeout
-    #   gitpython
-    #   importlib-metadata
-    #   pytest-shell-utilities
-    #   pytest-system-statistics
-    #   yarl
-urllib3==1.26.6
-    # via
-    #   -r requirements/windows.txt
-    #   botocore
-    #   kubernetes
-    #   python-etcd
-    #   requests
-virtualenv==20.7.2
-    # via
-    #   -r requirements/static/ci/common.in
-    #   pytest-salt-factories
-watchdog==0.10.3
-    # via -r requirements/static/ci/common.in
-websocket-client==0.40.0
-    # via
-    #   docker
-    #   kubernetes
-werkzeug==2.2.3
-    # via
-    #   moto
-    #   pytest-httpserver
-wheel==0.38.4
-    # via -r requirements/windows.txt
-wmi==1.5.1
-    # via -r requirements/windows.txt
-xmltodict==0.12.0
-    # via
-    #   moto
-    #   pywinrm
-yamllint==1.26.3
-    # via -r requirements/static/ci/windows.in
-yarl==1.7.2
-    # via aiohttp
-zc.lockfile==2.0
-    # via cherrypy
-zipp==3.5.0
-    # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/requirements/static/pkg/py3.7/freebsd.txt b/requirements/static/pkg/py3.7/freebsd.txt
deleted file mode 100644
index 6b65557c888..00000000000
--- a/requirements/static/pkg/py3.7/freebsd.txt
+++ /dev/null
@@ -1,107 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/pkg/py3.7/freebsd.txt requirements/base.txt requirements/static/pkg/freebsd.in requirements/zeromq.txt
-#
-certifi==2022.12.7
-    # via requests
-cffi==1.14.6
-    # via cryptography
-charset-normalizer==3.1.0
-    # via requests
-cheroot==8.5.2
-    # via cherrypy
-cherrypy==18.6.1
-    # via -r requirements/static/pkg/freebsd.in
-contextvars==2.4
-    # via -r requirements/base.txt
-cryptography==39.0.2
-    # via pyopenssl
-distro==1.5.0
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/static/pkg/freebsd.in
-idna==2.8
-    # via requests
-immutables==0.15
-    # via contextvars
-importlib-metadata==4.6.4
-    # via -r requirements/static/pkg/freebsd.in
-jaraco.classes==3.2.1
-    # via jaraco.collections
-jaraco.collections==3.4.0
-    # via cherrypy
-jaraco.functools==2.0
-    # via
-    #   cheroot
-    #   jaraco.text
-    #   tempora
-jaraco.text==3.5.1
-    # via jaraco.collections
-jinja2==3.1.2
-    # via -r requirements/base.txt
-jmespath==1.0.1
-    # via -r requirements/base.txt
-looseversion==1.0.2
-    # via -r requirements/base.txt
-markupsafe==2.1.2
-    # via
-    #   -r requirements/base.txt
-    #   jinja2
-more-itertools==5.0.0
-    # via
-    #   cheroot
-    #   cherrypy
-    #   jaraco.classes
-    #   jaraco.functools
-msgpack==1.0.2
-    # via -r requirements/base.txt
-packaging==22.0
-    # via -r requirements/base.txt
-portend==2.4
-    # via cherrypy
-psutil==5.8.0
-    # via -r requirements/base.txt
-pycparser==2.17
-    # via cffi
-pycryptodomex==3.9.8
-    # via -r requirements/crypto.txt
-pyopenssl==23.0.0
-    # via -r requirements/static/pkg/freebsd.in
-python-dateutil==2.8.1
-    # via -r requirements/static/pkg/freebsd.in
-python-gnupg==0.4.8
-    # via -r requirements/static/pkg/freebsd.in
-pytz==2022.1
-    # via tempora
-pyyaml==5.4.1
-    # via -r requirements/base.txt
-pyzmq==23.2.0 ; python_version < "3.11"
-    # via -r requirements/zeromq.txt
-requests==2.31.0
-    # via -r requirements/base.txt
-setproctitle==1.3.2
-    # via -r requirements/static/pkg/freebsd.in
-six==1.16.0
-    # via
-    #   cheroot
-    #   more-itertools
-    #   python-dateutil
-tempora==4.1.1
-    # via portend
-timelib==0.2.5
-    # via -r requirements/static/pkg/freebsd.in
-tornado==6.2.0 ; python_version < "3.8"
-    # via -r requirements/base.txt
-typing-extensions==3.10.0.0
-    # via importlib-metadata
-urllib3==1.26.6
-    # via requests
-zc.lockfile==1.4
-    # via cherrypy
-zipp==3.5.0
-    # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/requirements/static/pkg/py3.7/linux.txt b/requirements/static/pkg/py3.7/linux.txt
deleted file mode 100644
index 4c419d97e70..00000000000
--- a/requirements/static/pkg/py3.7/linux.txt
+++ /dev/null
@@ -1,109 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/pkg/py3.7/linux.txt requirements/base.txt requirements/static/pkg/linux.in requirements/zeromq.txt
-#
-certifi==2022.12.7
-    # via requests
-cffi==1.14.6
-    # via cryptography
-charset-normalizer==3.1.0
-    # via requests
-cheroot==8.5.2
-    # via cherrypy
-cherrypy==18.6.1
-    # via -r requirements/static/pkg/linux.in
-contextvars==2.4
-    # via -r requirements/base.txt
-cryptography==39.0.2
-    # via
-    #   -r requirements/static/pkg/linux.in
-    #   pyopenssl
-distro==1.5.0
-    # via -r requirements/base.txt
-idna==2.8
-    # via requests
-immutables==0.15
-    # via contextvars
-importlib-metadata==4.6.4
-    # via -r requirements/static/pkg/linux.in
-jaraco.classes==3.2.1
-    # via jaraco.collections
-jaraco.collections==3.4.0
-    # via cherrypy
-jaraco.functools==2.0
-    # via
-    #   cheroot
-    #   jaraco.text
-    #   tempora
-jaraco.text==3.5.1
-    # via jaraco.collections
-jinja2==3.1.2
-    # via -r requirements/base.txt
-jmespath==1.0.1
-    # via -r requirements/base.txt
-looseversion==1.0.2
-    # via -r requirements/base.txt
-markupsafe==2.1.2
-    # via
-    #   -r requirements/base.txt
-    #   jinja2
-more-itertools==5.0.0
-    # via
-    #   cheroot
-    #   cherrypy
-    #   jaraco.classes
-    #   jaraco.functools
-msgpack==1.0.2
-    # via -r requirements/base.txt
-packaging==22.0
-    # via -r requirements/base.txt
-portend==2.4
-    # via cherrypy
-psutil==5.8.0
-    # via -r requirements/base.txt
-pycparser==2.17
-    # via cffi
-pycryptodomex==3.9.8
-    # via -r requirements/crypto.txt
-pyopenssl==23.0.0
-    # via -r requirements/static/pkg/linux.in
-python-dateutil==2.8.1
-    # via -r requirements/static/pkg/linux.in
-python-gnupg==0.4.8
-    # via -r requirements/static/pkg/linux.in
-pytz==2022.1
-    # via tempora
-pyyaml==5.4.1
-    # via -r requirements/base.txt
-pyzmq==23.2.0 ; python_version < "3.11"
-    # via -r requirements/zeromq.txt
-requests==2.31.0
-    # via -r requirements/base.txt
-rpm-vercmp==0.1.2
-    # via -r requirements/static/pkg/linux.in
-setproctitle==1.3.2
-    # via -r requirements/static/pkg/linux.in
-six==1.16.0
-    # via
-    #   cheroot
-    #   more-itertools
-    #   python-dateutil
-tempora==4.1.1
-    # via portend
-timelib==0.2.5
-    # via -r requirements/static/pkg/linux.in
-tornado==6.2.0 ; python_version < "3.8"
-    # via -r requirements/base.txt
-typing-extensions==3.10.0.0
-    # via importlib-metadata
-urllib3==1.26.6
-    # via requests
-zc.lockfile==1.4
-    # via cherrypy
-zipp==3.5.0
-    # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/requirements/static/pkg/py3.7/windows.txt b/requirements/static/pkg/py3.7/windows.txt
deleted file mode 100644
index 2456b246175..00000000000
--- a/requirements/static/pkg/py3.7/windows.txt
+++ /dev/null
@@ -1,148 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=requirements/static/pkg/py3.7/windows.txt requirements/static/pkg/windows.in requirements/windows.txt
-#
-certifi==2022.12.7
-    # via
-    #   -r requirements/windows.txt
-    #   requests
-cffi==1.14.6
-    # via
-    #   -r requirements/windows.txt
-    #   clr-loader
-    #   cryptography
-charset-normalizer==3.1.0
-    # via requests
-cheroot==8.5.2
-    # via cherrypy
-cherrypy==18.6.1
-    # via -r requirements/windows.txt
-clr-loader==0.2.4
-    # via pythonnet
-contextvars==2.4
-    # via -r requirements/base.txt
-cryptography==39.0.2
-    # via
-    #   -r requirements/windows.txt
-    #   pyopenssl
-distro==1.5.0
-    # via -r requirements/base.txt
-gitdb==4.0.7
-    # via gitpython
-gitpython==3.1.30 ; python_version >= "3.7"
-    # via -r requirements/windows.txt
-idna==2.8
-    # via requests
-immutables==0.15
-    # via contextvars
-importlib-metadata==4.6.4
-    # via -r requirements/windows.txt
-ioloop==0.1a0
-    # via -r requirements/windows.txt
-jaraco.classes==3.2.1
-    # via jaraco.collections
-jaraco.collections==3.3.0
-    # via cherrypy
-jaraco.functools==2.0
-    # via
-    #   cheroot
-    #   jaraco.text
-    #   tempora
-jaraco.text==3.5.0
-    # via jaraco.collections
-jinja2==3.1.2
-    # via -r requirements/base.txt
-jmespath==1.0.1
-    # via -r requirements/base.txt
-looseversion==1.0.2
-    # via -r requirements/base.txt
-lxml==4.9.1
-    # via -r requirements/windows.txt
-markupsafe==2.1.2
-    # via
-    #   -r requirements/base.txt
-    #   jinja2
-more-itertools==8.2.0
-    # via
-    #   cheroot
-    #   cherrypy
-    #   jaraco.classes
-    #   jaraco.functools
-msgpack==1.0.2
-    # via -r requirements/base.txt
-packaging==22.0
-    # via -r requirements/base.txt
-portend==2.6
-    # via cherrypy
-psutil==5.8.0
-    # via -r requirements/base.txt
-pyasn1==0.4.8
-    # via -r requirements/windows.txt
-pycparser==2.21
-    # via
-    #   -r requirements/windows.txt
-    #   cffi
-pycryptodomex==3.10.1
-    # via -r requirements/crypto.txt
-pymssql==2.2.1
-    # via -r requirements/windows.txt
-pymysql==1.0.2
-    # via -r requirements/windows.txt
-pyopenssl==23.0.0
-    # via -r requirements/windows.txt
-python-dateutil==2.8.1
-    # via -r requirements/windows.txt
-python-gnupg==0.4.8
-    # via -r requirements/windows.txt
-pythonnet==3.0.1
-    # via -r requirements/windows.txt
-pytz==2022.1
-    # via tempora
-pywin32==305
-    # via
-    #   -r requirements/windows.txt
-    #   cherrypy
-    #   wmi
-pyyaml==5.4.1
-    # via -r requirements/base.txt
-pyzmq==25.0.2 ; sys_platform == "win32"
-    # via -r requirements/zeromq.txt
-requests==2.31.0
-    # via
-    #   -r requirements/base.txt
-    #   -r requirements/windows.txt
-setproctitle==1.3.2
-    # via -r requirements/windows.txt
-six==1.15.0
-    # via
-    #   cheroot
-    #   python-dateutil
-smmap==4.0.0
-    # via gitdb
-tempora==4.1.1
-    # via portend
-timelib==0.2.5
-    # via -r requirements/windows.txt
-tornado==6.2.0 ; python_version < "3.8"
-    # via -r requirements/base.txt
-typing-extensions==4.4.0
-    # via
-    #   gitpython
-    #   importlib-metadata
-urllib3==1.26.6
-    # via
-    #   -r requirements/windows.txt
-    #   requests
-wheel==0.38.4
-    # via -r requirements/windows.txt
-wmi==1.5.1
-    # via -r requirements/windows.txt
-zc.lockfile==2.0
-    # via cherrypy
-zipp==3.5.0
-    # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/salt/_logging/handlers.py b/salt/_logging/handlers.py
index f4b0b6fec3d..5c550b565f8 100644
--- a/salt/_logging/handlers.py
+++ b/salt/_logging/handlers.py
@@ -5,7 +5,6 @@
     Salt's logging handlers
 """
 
-import copy
 import logging
 import logging.handlers
 import queue as _queue
@@ -147,7 +146,7 @@ class SysLogHandler(ExcInfoOnLogLevelFormatMixin, logging.handlers.SysLogHandler
         Deal with syslog os errors when the log file does not exist
         """
         handled = False
-        if sys.stderr and sys.version_info >= (3, 5, 4):
+        if sys.stderr:
             exc_type, exc, exc_traceback = sys.exc_info()
             try:
                 if exc_type.__name__ in "FileNotFoundError":
@@ -216,92 +215,31 @@ class WatchedFileHandler(
     """
 
 
-if sys.version_info < (3, 7):
-    # On python versions lower than 3.7, we sill subclass and overwrite prepare to include the fix for:
-    #  https://bugs.python.org/issue35726
-    class QueueHandler(
-        ExcInfoOnLogLevelFormatMixin, logging.handlers.QueueHandler
-    ):  # pylint: disable=no-member,inconsistent-mro
-        def __init__(self, queue):  # pylint: disable=useless-super-delegation
-            super().__init__(queue)
-            warn_until_date(
-                "20240101",
-                "Please stop using '{name}.QueueHandler' and instead "
-                "use 'logging.handlers.QueueHandler'. "
-                "'{name}.QueueHandler' will go away after "
-                "{{date}}.".format(name=__name__),
+class QueueHandler(
+    ExcInfoOnLogLevelFormatMixin, logging.handlers.QueueHandler
+):  # pylint: disable=no-member,inconsistent-mro
+    def __init__(self, queue):  # pylint: disable=useless-super-delegation
+        super().__init__(queue)
+        warn_until_date(
+            "20240101",
+            "Please stop using '{name}.QueueHandler' and instead "
+            "use 'logging.handlers.QueueHandler'. "
+            "'{name}.QueueHandler' will go away after "
+            "{{date}}.".format(name=__name__),
+        )
+
+    def enqueue(self, record):
+        """
+        Enqueue a record.
+
+        The base implementation uses put_nowait. You may want to override
+        this method if you want to use blocking, timeouts or custom queue
+        implementations.
+        """
+        try:
+            self.queue.put_nowait(record)
+        except _queue.Full:
+            sys.stderr.write(
+                "[WARNING ] Message queue is full, "
+                'unable to write "{}" to log.\n'.format(record)
             )
-
-        def enqueue(self, record):
-            """
-            Enqueue a record.
-
-            The base implementation uses put_nowait. You may want to override
-            this method if you want to use blocking, timeouts or custom queue
-            implementations.
-            """
-            try:
-                self.queue.put_nowait(record)
-            except _queue.Full:
-                sys.stderr.write(
-                    "[WARNING ] Message queue is full, "
-                    'unable to write "{}" to log.\n'.format(record)
-                )
-
-        def prepare(self, record):
-            """
-            Prepares a record for queuing. The object returned by this method is
-            enqueued.
-            The base implementation formats the record to merge the message
-            and arguments, and removes unpickleable items from the record
-            in-place.
-            You might want to override this method if you want to convert
-            the record to a dict or JSON string, or send a modified copy
-            of the record while leaving the original intact.
-            """
-            # The format operation gets traceback text into record.exc_text
-            # (if there's exception data), and also returns the formatted
-            # message. We can then use this to replace the original
-            # msg + args, as these might be unpickleable. We also zap the
-            # exc_info and exc_text attributes, as they are no longer
-            # needed and, if not None, will typically not be pickleable.
-            msg = self.format(record)
-            # bpo-35726: make copy of record to avoid affecting other handlers in the chain.
-            record = copy.copy(record)
-            record.message = msg
-            record.msg = msg
-            record.args = None
-            record.exc_info = None
-            record.exc_text = None
-            return record
-
-else:
-
-    class QueueHandler(
-        ExcInfoOnLogLevelFormatMixin, logging.handlers.QueueHandler
-    ):  # pylint: disable=no-member,inconsistent-mro
-        def __init__(self, queue):  # pylint: disable=useless-super-delegation
-            super().__init__(queue)
-            warn_until_date(
-                "20240101",
-                "Please stop using '{name}.QueueHandler' and instead "
-                "use 'logging.handlers.QueueHandler'. "
-                "'{name}.QueueHandler' will go away after "
-                "{{date}}.".format(name=__name__),
-            )
-
-        def enqueue(self, record):
-            """
-            Enqueue a record.
-
-            The base implementation uses put_nowait. You may want to override
-            this method if you want to use blocking, timeouts or custom queue
-            implementations.
-            """
-            try:
-                self.queue.put_nowait(record)
-            except _queue.Full:
-                sys.stderr.write(
-                    "[WARNING ] Message queue is full, "
-                    'unable to write "{}" to log.\n'.format(record)
-                )
diff --git a/salt/utils/idem.py b/salt/utils/idem.py
index ab7641a8e91..5c14cf7c47a 100644
--- a/salt/utils/idem.py
+++ b/salt/utils/idem.py
@@ -7,7 +7,6 @@ This util provides access to an idem-ready hub
 .. versionadded:: 3002
 """
 import logging
-import sys
 
 try:
     import pop.hub
@@ -22,8 +21,6 @@ __virtualname__ = "idem"
 
 
 def __virtual__():
-    if sys.version_info < (3, 6):
-        return False, "idem only works on python3.6 and later"
     if not HAS_POP[0]:
         return HAS_POP
     return __virtualname__
diff --git a/salt/utils/templates.py b/salt/utils/templates.py
index 4a8adf2a14f..5e477207b86 100644
--- a/salt/utils/templates.py
+++ b/salt/utils/templates.py
@@ -2,12 +2,14 @@
 Template render systems
 """
 import codecs
+import importlib.machinery
+import importlib.util
 import logging
 import os
+import pathlib
 import sys
 import tempfile
 import traceback
-from pathlib import Path
 
 import jinja2
 import jinja2.ext
@@ -31,17 +33,6 @@ from salt.utils.decorators.jinja import JinjaFilter, JinjaGlobal, JinjaTest
 from salt.utils.odict import OrderedDict
 from salt.utils.versions import Version
 
-if sys.version_info[:2] >= (3, 5):
-    import importlib.machinery  # pylint: disable=no-name-in-module,import-error
-    import importlib.util  # pylint: disable=no-name-in-module,import-error
-
-    USE_IMPORTLIB = True
-else:
-    import imp
-
-    USE_IMPORTLIB = False
-
-
 log = logging.getLogger(__name__)
 
 
@@ -118,14 +109,14 @@ def generate_sls_context(tmplpath, sls):
 
     if tmplpath:
         # Normalize template path
-        template = str(Path(tmplpath).as_posix())
+        template = str(pathlib.Path(tmplpath).as_posix())
 
         # Determine proper template name without root
         if not sls:
             template = template.rsplit("/", 1)[-1]
-        elif template.endswith("{}.sls".format(slspath)):
+        elif template.endswith(f"{slspath}.sls"):
             template = template[-(4 + len(slspath)) :]
-        elif template.endswith("{}/init.sls".format(slspath)):
+        elif template.endswith(f"{slspath}/init.sls"):
             template = template[-(9 + len(slspath)) :]
         else:
             # Something went wrong
@@ -322,14 +313,14 @@ def _get_jinja_error(trace, context=None):
     # error log place at the beginning
     if add_log:
         if template_path:
-            out = "\n{}\n".format(msg.splitlines()[0])
+            out = f"\n{msg.splitlines()[0]}\n"
             with salt.utils.files.fopen(template_path) as fp_:
                 template_contents = salt.utils.stringutils.to_unicode(fp_.read())
             out += salt.utils.stringutils.get_context(
                 template_contents, line, marker="    <======================"
             )
         else:
-            out = "\n{}\n".format(msg)
+            out = f"\n{msg}\n"
         line = 0
     return line, out
 
@@ -479,7 +470,7 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None):
             line, out = _get_jinja_error(trace, context=decoded_context)
             if not line:
                 tmplstr = ""
-            raise SaltRenderError("Jinja variable {}{}".format(exc, out), line, tmplstr)
+            raise SaltRenderError(f"Jinja variable {exc}{out}", line, tmplstr)
         except (
             jinja2.exceptions.TemplateRuntimeError,
             jinja2.exceptions.TemplateSyntaxError,
@@ -489,9 +480,7 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None):
             line, out = _get_jinja_error(trace, context=decoded_context)
             if not line:
                 tmplstr = ""
-            raise SaltRenderError(
-                "Jinja syntax error: {}{}".format(exc, out), line, tmplstr
-            )
+            raise SaltRenderError(f"Jinja syntax error: {exc}{out}", line, tmplstr)
         except (SaltInvocationError, CommandExecutionError) as exc:
             trace = traceback.extract_tb(sys.exc_info()[2])
             line, out = _get_jinja_error(trace, context=decoded_context)
@@ -511,7 +500,7 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None):
             if not line:
                 tmplstr = ""
             else:
-                tmplstr += "\n{}".format(tracestr)
+                tmplstr += f"\n{tracestr}"
             log.debug("Jinja Error")
             log.debug("Exception:", exc_info=True)
             log.debug("Out: %s", out)
@@ -520,7 +509,7 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None):
             log.debug("TraceStr: %s", tracestr)
 
             raise SaltRenderError(
-                "Jinja error: {}{}".format(exc, out), line, tmplstr, trace=tracestr
+                f"Jinja error: {exc}{out}", line, tmplstr, trace=tracestr
             )
     finally:
         if loader and isinstance(loader, salt.utils.jinja.SaltCacheLoader):
@@ -676,25 +665,21 @@ def py(sfn, string=False, **kwargs):  # pylint: disable=C0103
     base_fname = os.path.basename(sfn)
     name = base_fname.split(".")[0]
 
-    if USE_IMPORTLIB:
-        # pylint: disable=no-member
-        loader = importlib.machinery.SourceFileLoader(name, sfn)
-        spec = importlib.util.spec_from_file_location(name, sfn, loader=loader)
-        if spec is None:
-            raise ImportError()
-        mod = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(mod)
-        # pylint: enable=no-member
-        sys.modules[name] = mod
-    else:
-        mod = imp.load_source(name, sfn)
+    loader = importlib.machinery.SourceFileLoader(name, sfn)
+    spec = importlib.util.spec_from_file_location(name, sfn, loader=loader)
+    if spec is None:
+        raise ImportError()
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    # pylint: enable=no-member
+    sys.modules[name] = mod
 
     # File templates need these set as __var__
     if "__env__" not in kwargs and "saltenv" in kwargs:
         setattr(mod, "__env__", kwargs["saltenv"])
         builtins = ["salt", "grains", "pillar", "opts"]
         for builtin in builtins:
-            arg = "__{}__".format(builtin)
+            arg = f"__{builtin}__"
             setattr(mod, arg, kwargs[builtin])
 
     for kwarg in kwargs:
diff --git a/setup.py b/setup.py
index a6f5fef0a68..0f5870e9029 100755
--- a/setup.py
+++ b/setup.py
@@ -485,7 +485,7 @@ class CloudSdist(Sdist):  # pylint: disable=too-many-ancestors
                 with open(deploy_path, "w") as fp_:
                     fp_.write(script_contents)
             except OSError as err:
-                log.error("Failed to write the updated script: {}".format(err))
+                log.error(f"Failed to write the updated script: {err}")
 
         # Let's the rest of the build command
         Sdist.run(self)
@@ -544,7 +544,7 @@ class Clean(clean):
         for subdir in ("salt", "tests", "doc"):
             root = os.path.join(os.path.dirname(__file__), subdir)
             for dirname, _, _ in os.walk(root):
-                for to_remove_filename in glob.glob("{}/*.py[oc]".format(dirname)):
+                for to_remove_filename in glob.glob(f"{dirname}/*.py[oc]"):
                     os.remove(to_remove_filename)
 
 
@@ -849,14 +849,12 @@ class SaltDistribution(distutils.dist.Distribution):
         with open(SALT_LONG_DESCRIPTION_FILE, encoding="utf-8") as f:
             self.long_description = f.read()
         self.long_description_content_type = "text/x-rst"
-        self.python_requires = ">=3.6"
+        self.python_requires = ">=3.8"
         self.classifiers = [
             "Programming Language :: Python",
             "Programming Language :: Cython",
             "Programming Language :: Python :: 3",
             "Programming Language :: Python :: 3 :: Only",
-            "Programming Language :: Python :: 3.6",
-            "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
             "Programming Language :: Python :: 3.9",
             "Programming Language :: Python :: 3.10",
@@ -910,8 +908,8 @@ class SaltDistribution(distutils.dist.Distribution):
                 continue
             if attrname == "salt_version":
                 attrname = "version"
-            if hasattr(self.metadata, "set_{}".format(attrname)):
-                getattr(self.metadata, "set_{}".format(attrname))(attrvalue)
+            if hasattr(self.metadata, f"set_{attrname}"):
+                getattr(self.metadata, f"set_{attrname}")(attrvalue)
             elif hasattr(self.metadata, attrname):
                 try:
                     setattr(self.metadata, attrname, attrvalue)
diff --git a/tests/conftest.py b/tests/conftest.py
index a7777c2cea6..51a5c04d82b 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -408,7 +408,7 @@ def set_max_open_files_limits(min_soft=3072, min_hard=4096):
 
 def pytest_report_header():
     soft, hard = set_max_open_files_limits()
-    return "max open files; soft: {}; hard: {}".format(soft, hard)
+    return f"max open files; soft: {soft}; hard: {hard}"
 
 
 def pytest_itemcollected(item):
@@ -850,7 +850,7 @@ def groups_collection_modifyitems(config, items):
 
     terminal_reporter = config.pluginmanager.get_plugin("terminalreporter")
     terminal_reporter.write(
-        "Running test group #{} ({} tests)\n".format(group_id, len(items)),
+        f"Running test group #{group_id} ({len(items)} tests)\n",
         yellow=True,
     )
 
@@ -911,12 +911,7 @@ def integration_files_dir(salt_factories):
     for child in (PYTESTS_DIR / "integration" / "files").iterdir():
         destpath = dirname / child.name
         if child.is_dir():
-            if sys.version_info >= (3, 8):
-                shutil.copytree(str(child), str(destpath), dirs_exist_ok=True)
-            else:
-                if destpath.exists():
-                    shutil.rmtree(str(destpath), ignore_errors=True)
-                shutil.copytree(str(child), str(destpath))
+            shutil.copytree(str(child), str(destpath), dirs_exist_ok=True)
         else:
             shutil.copyfile(str(child), str(destpath))
     return dirname
@@ -1500,7 +1495,7 @@ def from_filenames_collection_modifyitems(config, items):
             path.replace("\\", os.sep).replace("/", os.sep)
         )
         if not properly_slashed_path.exists():
-            errors.append("{}: Does not exist".format(properly_slashed_path))
+            errors.append(f"{properly_slashed_path}: Does not exist")
             continue
         if (
             properly_slashed_path.name == "testrun-changed-files.txt"
@@ -1523,12 +1518,12 @@ def from_filenames_collection_modifyitems(config, items):
                     )
                     continue
                 changed_files_selections.append(
-                    "{}: Source {}".format(line_path, properly_slashed_path)
+                    f"{line_path}: Source {properly_slashed_path}"
                 )
                 from_filenames_paths.add(line_path)
             continue
         changed_files_selections.append(
-            "{}: Source --from-filenames".format(properly_slashed_path)
+            f"{properly_slashed_path}: Source --from-filenames"
         )
         from_filenames_paths.add(properly_slashed_path)
 
@@ -1558,7 +1553,7 @@ def from_filenames_collection_modifyitems(config, items):
                 continue
             # Tests in the listing don't require additional matching and will be added to the
             # list of tests to run
-            test_module_selections.append("{}: Source --from-filenames".format(path))
+            test_module_selections.append(f"{path}: Source --from-filenames")
             test_module_paths.add(path)
             continue
         if path.name == "setup.py" or path.as_posix().startswith("salt/"):
@@ -1571,18 +1566,18 @@ def from_filenames_collection_modifyitems(config, items):
                 # salt/version.py ->
                 #    tests/unit/test_version.py
                 #    tests/pytests/unit/test_version.py
-                "**/test_{}".format(path.name),
+                f"**/test_{path.name}",
                 # salt/modules/grains.py ->
                 #    tests/pytests/integration/modules/grains/tests_*.py
                 # salt/modules/saltutil.py ->
                 #    tests/pytests/integration/modules/saltutil/test_*.py
-                "**/{}/test_*.py".format(path.stem),
+                f"**/{path.stem}/test_*.py",
                 # salt/modules/config.py ->
                 #    tests/unit/modules/test_config.py
                 #    tests/integration/modules/test_config.py
                 #    tests/pytests/unit/modules/test_config.py
                 #    tests/pytests/integration/modules/test_config.py
-                "**/{}/test_{}".format(path.parent.name, path.name),
+                f"**/{path.parent.name}/test_{path.name}",
             )
             for pattern in glob_patterns:
                 for match in TESTS_DIR.rglob(pattern):
@@ -1641,20 +1636,20 @@ def from_filenames_collection_modifyitems(config, items):
                         test_module_paths.add(match_path)
             continue
         else:
-            errors.append("{}: Don't know what to do with this path".format(path))
+            errors.append(f"{path}: Don't know what to do with this path")
 
     if errors:
         terminal_reporter.write("Errors:\n", bold=True)
         for error in errors:
-            terminal_reporter.write(" * {}\n".format(error))
+            terminal_reporter.write(f" * {error}\n")
     if changed_files_selections:
         terminal_reporter.write("Changed files collected:\n", bold=True)
         for selection in changed_files_selections:
-            terminal_reporter.write(" * {}\n".format(selection))
+            terminal_reporter.write(f" * {selection}\n")
     if test_module_selections:
         terminal_reporter.write("Selected test modules:\n", bold=True)
         for selection in test_module_selections:
-            terminal_reporter.write(" * {}\n".format(selection))
+            terminal_reporter.write(f" * {selection}\n")
     terminal_reporter.section(
         "From Filenames(--from-filenames) Test Selection", sep="<"
     )
diff --git a/tests/pytests/unit/modules/test_genesis.py b/tests/pytests/unit/modules/test_genesis.py
index 0a2100f4848..7a3b1632843 100644
--- a/tests/pytests/unit/modules/test_genesis.py
+++ b/tests/pytests/unit/modules/test_genesis.py
@@ -1,10 +1,6 @@
 """
     :codeauthor: Rupesh Tare 
 """
-
-
-import sys
-
 import pytest
 
 import salt.modules.genesis as genesis
@@ -20,11 +16,7 @@ def test_bootstrap():
     """
     Test for Create an image for a specific platform.
     """
-    # Changed in 3.7.0 pformat no longer includes the comma
-    if sys.version_info >= (3, 7):
-        exception_string = "Exception({})".format(repr("foo"))
-    else:
-        exception_string = "Exception({},)".format(repr("foo"))
+    exception_string = "Exception({})".format(repr("foo"))
     mock = MagicMock(return_value=False)
     with patch.dict(genesis.__salt__, {"file.directory_exists": mock}):
         mock = MagicMock(side_effect=Exception("foo"))
diff --git a/tests/pytests/unit/utils/test_stringutils.py b/tests/pytests/unit/utils/test_stringutils.py
index cece5a08494..5b26d3473b1 100644
--- a/tests/pytests/unit/utils/test_stringutils.py
+++ b/tests/pytests/unit/utils/test_stringutils.py
@@ -4,7 +4,6 @@ Tests for stringutils utility file.
 
 import builtins
 import re
-import sys
 import textwrap
 
 import pytest
@@ -185,7 +184,7 @@ def test_is_binary():
     assert salt.utils.stringutils.is_binary(b"") is False
 
     nontext = 3 * "".join([chr(x) for x in range(1, 32) if x not in (8, 9, 10, 12, 13)])
-    almost_bin_str = "{}{}".format(LOREM_IPSUM[:100], nontext[:42])
+    almost_bin_str = f"{LOREM_IPSUM[:100]}{nontext[:42]}"
 
     assert salt.utils.stringutils.is_binary(almost_bin_str) is False
     # Also test bytestring
@@ -278,16 +277,10 @@ def test_build_whitespace_split_regex():
     # With 3.7+,  re.escape only escapes special characters, no longer
     # escaping all characters other than ASCII letters, numbers and
     # underscores.  This includes commas.
-    if sys.version_info >= (3, 7):
-        expected_regex = (
-            "(?m)^(?:[\\s]+)?Lorem(?:[\\s]+)?ipsum(?:[\\s]+)?dolor(?:[\\s]+)?sit(?:[\\s]+)?amet,"
-            "(?:[\\s]+)?$"
-        )
-    else:
-        expected_regex = (
-            "(?m)^(?:[\\s]+)?Lorem(?:[\\s]+)?ipsum(?:[\\s]+)?dolor(?:[\\s]+)?sit(?:[\\s]+)?amet\\,"
-            "(?:[\\s]+)?$"
-        )
+    expected_regex = (
+        "(?m)^(?:[\\s]+)?Lorem(?:[\\s]+)?ipsum(?:[\\s]+)?dolor(?:[\\s]+)?sit(?:[\\s]+)?amet,"
+        "(?:[\\s]+)?$"
+    )
     assert (
         salt.utils.stringutils.build_whitespace_split_regex(
             " ".join(LOREM_IPSUM.split()[:5])
@@ -748,24 +741,15 @@ def test_human_to_bytes(unit):
 
     for val in vals:
         # calculate KB, MB, GB, etc. as 1024 instead of 1000 (legacy use)
-        assert (
-            salt.utils.stringutils.human_to_bytes("{}{}".format(val, unit)) == val * iec
-        )
-        assert (
-            salt.utils.stringutils.human_to_bytes("{} {}".format(val, unit))
-            == val * iec
-        )
+        assert salt.utils.stringutils.human_to_bytes(f"{val}{unit}") == val * iec
+        assert salt.utils.stringutils.human_to_bytes(f"{val} {unit}") == val * iec
         # handle metric (KB, MB, GB, etc.) per standard
         assert (
-            salt.utils.stringutils.human_to_bytes(
-                "{}{}".format(val, unit), handle_metric=True
-            )
+            salt.utils.stringutils.human_to_bytes(f"{val}{unit}", handle_metric=True)
             == val * multiplier
         )
         assert (
-            salt.utils.stringutils.human_to_bytes(
-                "{} {}".format(val, unit), handle_metric=True
-            )
+            salt.utils.stringutils.human_to_bytes(f"{val} {unit}", handle_metric=True)
             == val * multiplier
         )
 

From 20ac30d1e5ac4c46ed8b5a657f3eac1fe3d3e528 Mon Sep 17 00:00:00 2001
From: jeanluc 
Date: Thu, 8 Dec 2022 19:53:01 +0100
Subject: [PATCH 142/152] Add tests for issue 52164

---
 tests/pytests/integration/master/test_peer.py | 130 ++++++++++++++++++
 1 file changed, 130 insertions(+)
 create mode 100644 tests/pytests/integration/master/test_peer.py

diff --git a/tests/pytests/integration/master/test_peer.py b/tests/pytests/integration/master/test_peer.py
new file mode 100644
index 00000000000..8ac9fdef4e7
--- /dev/null
+++ b/tests/pytests/integration/master/test_peer.py
@@ -0,0 +1,130 @@
+import shutil
+
+import pytest
+from saltfactories.utils import random_string
+
+
+@pytest.fixture(scope="module")
+def pillar_state_tree(tmp_path_factory):
+    _pillar_state_tree = tmp_path_factory.mktemp("pillar")
+    try:
+        yield _pillar_state_tree
+    finally:
+        shutil.rmtree(str(_pillar_state_tree), ignore_errors=True)
+
+
+@pytest.fixture(scope="module")
+def peer_salt_master_config(pillar_state_tree):
+    return {
+        "pillar_roots": {"base": [str(pillar_state_tree)]},
+        "open_mode": True,
+        "peer": {
+            ".*": ["test.ping"],
+            "peer-comm-minion.*": [
+                {
+                    "G@hello_peer:beer": ["grains.get"],
+                }
+            ],
+        },
+    }
+
+
+@pytest.fixture(scope="module")
+def peer_salt_master(
+    salt_factories, pillar_state_tree, vault_port, peer_salt_master_config
+):
+    factory = salt_factories.salt_master_daemon(
+        random_string("peer-comm-master", uppercase=False),
+        defaults=peer_salt_master_config,
+    )
+    with factory.started():
+        yield factory
+
+
+@pytest.fixture(scope="module")
+def peer_salt_minion_1(peer_salt_master):
+    assert peer_salt_master.is_running()
+    factory = peer_salt_master.salt_minion_daemon(
+        random_string("peer-comm-minion-1", uppercase=False),
+        defaults={"open_mode": True, "grains": {"hello_peer": "beer"}},
+    )
+    with factory.started():
+        # Sync All
+        salt_call_cli = factory.salt_call_cli()
+        ret = salt_call_cli.run("saltutil.sync_all", _timeout=120)
+        assert ret.returncode == 0, ret
+        yield factory
+
+
+@pytest.fixture(scope="module")
+def peer_salt_minion_2(peer_salt_master):
+    assert peer_salt_master.is_running()
+    factory = peer_salt_master.salt_minion_daemon(
+        random_string("peer-comm-minion-2", uppercase=False),
+        defaults={"open_mode": True},
+    )
+    with factory.started():
+        # Sync All
+        salt_call_cli = factory.salt_call_cli()
+        ret = salt_call_cli.run("saltutil.sync_all", _timeout=120)
+        assert ret.returncode == 0, ret
+        yield factory
+
+
+@pytest.fixture(scope="module")
+def peer_salt_minion_3(peer_salt_master):
+    assert peer_salt_master.is_running()
+    factory = peer_salt_master.salt_minion_daemon(
+        random_string("peer-comm-minion-3", uppercase=False),
+        defaults={"open_mode": True},
+    )
+    with factory.started():
+        # Sync All
+        salt_call_cli = factory.salt_call_cli()
+        ret = salt_call_cli.run("saltutil.sync_all", _timeout=120)
+        assert ret.returncode == 0, ret
+        yield factory
+
+
+@pytest.mark.parametrize(
+    "source,target", ((x, y) for x in range(1, 4) for y in range(1, 4) if x != y)
+)
+def test_peer_communication(source, target, request):
+    cli = request.getfixturevalue(f"peer_salt_minion_{source}").salt_call_cli()
+    tgt = request.getfixturevalue(f"peer_salt_minion_{target}").id
+    ret = cli.run("publish.publish", tgt, "test.ping")
+    assert ret.returncode == 0
+    assert ret.data
+    assert tgt in ret.data
+    assert ret.data[tgt] is True
+
+
+def test_peer_communication_denied(peer_salt_minion_1, peer_salt_minion_2):
+    tgt = peer_salt_minion_2.id
+    ret = peer_salt_minion_1.salt_call_cli().run(
+        "publish.publish", tgt, "cmd.run", "echo pwned"
+    )
+    assert ret.returncode == 0
+    assert ret.data == {}
+
+
+@pytest.mark.parametrize("source", [2, 3])
+def test_peer_communication_limited_target_allowed(source, peer_salt_minion_1, request):
+    cli = request.getfixturevalue(f"peer_salt_minion_{source}").salt_call_cli()
+    tgt = peer_salt_minion_1.id
+    ret = cli.run("publish.publish", tgt, "grains.get", "hello_peer")
+    assert ret.returncode == 0
+    assert ret.data
+    assert tgt in ret.data
+    assert ret.data[tgt] == "beer"
+
+
+@pytest.mark.parametrize(
+    "source,target", ((x, y) for x in range(1, 4) for y in range(2, 4) if x != y)
+)
+def test_peer_communication_limited_target_denied(source, target, request):
+    cli = request.getfixturevalue(f"peer_salt_minion_{source}").salt_call_cli()
+    tgt = request.getfixturevalue(f"peer_salt_minion_{target}").id
+    ret = cli.run("publish.publish", tgt, "grains.get", "hello_peer")
+    assert ret.returncode == 0
+    assert ret.data == {}

From 278368b908e575983e902292f0ea098772eec4da Mon Sep 17 00:00:00 2001
From: jeanluc 
Date: Thu, 8 Dec 2022 20:13:24 +0100
Subject: [PATCH 143/152] Correct peer communication docs

---
 changelog/52164.fixed            |  1 +
 doc/ref/configuration/master.rst | 27 ++++++++++++------
 doc/ref/peer.rst                 | 49 +++++++++++++++++++++++---------
 3 files changed, 54 insertions(+), 23 deletions(-)
 create mode 100644 changelog/52164.fixed

diff --git a/changelog/52164.fixed b/changelog/52164.fixed
new file mode 100644
index 00000000000..ad7e7e7aead
--- /dev/null
+++ b/changelog/52164.fixed
@@ -0,0 +1 @@
+Corrected peer communication docs regarding target limiting
diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst
index bd1ccdde2e1..7a9f57ce9d3 100644
--- a/doc/ref/configuration/master.rst
+++ b/doc/ref/configuration/master.rst
@@ -5346,9 +5346,9 @@ and pkg modules.
 .. code-block:: yaml
 
     peer:
-      foo.example.com:
-          - test.*
-          - pkg.*
+      foo\.example\.com:
+          - test\..*
+          - pkg\..*
 
 This will allow all minions to execute all commands:
 
@@ -5361,16 +5361,25 @@ This will allow all minions to execute all commands:
 This is not recommended, since it would allow anyone who gets root on any
 single minion to instantly have root on all of the minions!
 
-By adding an additional layer you can limit the target hosts in addition to the
-accessible commands:
+It is also possible to limit target hosts with the :term:`Compound Matcher`.
+You can achieve this by adding another layer in between the source and the
+allowed functions:
 
 .. code-block:: yaml
 
     peer:
-      foo.example.com:
-        'db*':
-          - test.*
-          - pkg.*
+      '.*\.example\.com':
+        - 'G@role:db':
+          - test\..*
+          - pkg\..*
+
+.. note::
+
+    Notice that the source hosts are matched by a regular expression
+    on their minion ID, while target hosts can be matched by any of
+    the :ref:`available matchers `.
+
+    Note that globbing and regex matching on pillar values is not supported.
 
 .. conf_master:: peer_run
 
diff --git a/doc/ref/peer.rst b/doc/ref/peer.rst
index 3e4068d93bb..39efb123f9a 100644
--- a/doc/ref/peer.rst
+++ b/doc/ref/peer.rst
@@ -38,16 +38,16 @@ only recommended for very secure environments.
       .*:
         - .*
 
-This configuration will allow minions with IDs ending in example.com access
+This configuration will allow minions with IDs ending in ``.example.com`` access
 to the test, ps, and pkg module functions.
 
 .. code-block:: yaml
 
     peer:
-      .*example.com:
-        - test.*
-        - ps.*
-        - pkg.*
+      .*\.example.com:
+        - test\..*
+        - ps\..*
+        - pkg\..*
 
 
 The configuration logic is simple, a regular expression is passed for matching
@@ -58,17 +58,38 @@ allow minions ending with foo.org access to the publisher.
 .. code-block:: yaml
 
     peer:
-      .*example.com:
-        - test.*
-        - ps.*
-        - pkg.*
-      .*foo.org:
-        - test.*
-        - ps.*
-        - pkg.*
+      .*\.example.com:
+        - test\..*
+        - ps\..*
+        - pkg\..*
+      .*\.foo.org:
+        - test\..*
+        - ps\..*
+        - pkg\..*
 
 .. note::
-    Functions are matched using regular expressions.
+    Functions are matched using regular expressions as well.
+
+It is also possible to limit target hosts with the :term:`Compound Matcher`.
+You can achieve this by adding another layer in between the source and the
+allowed functions:
+
+.. code-block:: yaml
+
+    peer:
+      '.*\.example\.com':
+        - 'G@role:db':
+          - test\..*
+          - pkg\..*
+
+.. note::
+
+    Notice that the source hosts are matched by a regular expression
+    on their minion ID, while target hosts can be matched by any of
+    the :ref:`available matchers `.
+
+    Note that globbing and regex matching on pillar values is not supported.
+
 
 Peer Runner Communication
 =========================

From cc27e2a19a55fe20ee7400b688700847b8f409d2 Mon Sep 17 00:00:00 2001
From: jeanluc 
Date: Fri, 9 Dec 2022 08:30:59 +0100
Subject: [PATCH 144/152] Clarify that exact pillar matching is supported

---
 doc/ref/configuration/master.rst | 2 +-
 doc/ref/peer.rst                 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst
index 7a9f57ce9d3..3700ec84dd2 100644
--- a/doc/ref/configuration/master.rst
+++ b/doc/ref/configuration/master.rst
@@ -5379,7 +5379,7 @@ allowed functions:
     on their minion ID, while target hosts can be matched by any of
     the :ref:`available matchers `.
 
-    Note that globbing and regex matching on pillar values is not supported.
+    Note that globbing and regex matching on pillar values is not supported. You can only match exact values.
 
 .. conf_master:: peer_run
 
diff --git a/doc/ref/peer.rst b/doc/ref/peer.rst
index 39efb123f9a..035d749d91d 100644
--- a/doc/ref/peer.rst
+++ b/doc/ref/peer.rst
@@ -88,7 +88,7 @@ allowed functions:
     on their minion ID, while target hosts can be matched by any of
     the :ref:`available matchers `.
 
-    Note that globbing and regex matching on pillar values is not supported.
+    Note that globbing and regex matching on pillar values is not supported. You can only match exact values.
 
 
 Peer Runner Communication

From c15f469b4f6932229146f40049ca9847d9425ba7 Mon Sep 17 00:00:00 2001
From: jeanluc 
Date: Sun, 23 Apr 2023 23:55:13 +0200
Subject: [PATCH 145/152] Rename changelog files to .md

---
 changelog/{52164.fixed => 52164.fixed.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename changelog/{52164.fixed => 52164.fixed.md} (100%)

diff --git a/changelog/52164.fixed b/changelog/52164.fixed.md
similarity index 100%
rename from changelog/52164.fixed
rename to changelog/52164.fixed.md

From f5ce55dd4707419593a7ad7ffa4961f901ca3636 Mon Sep 17 00:00:00 2001
From: jeanluc 
Date: Wed, 31 May 2023 23:53:03 +0200
Subject: [PATCH 146/152] Address review comments

---
 changelog/52164.fixed.md | 1 -
 doc/ref/peer.rst         | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)
 delete mode 100644 changelog/52164.fixed.md

diff --git a/changelog/52164.fixed.md b/changelog/52164.fixed.md
deleted file mode 100644
index ad7e7e7aead..00000000000
--- a/changelog/52164.fixed.md
+++ /dev/null
@@ -1 +0,0 @@
-Corrected peer communication docs regarding target limiting
diff --git a/doc/ref/peer.rst b/doc/ref/peer.rst
index 035d749d91d..a39b3bc7ad7 100644
--- a/doc/ref/peer.rst
+++ b/doc/ref/peer.rst
@@ -38,7 +38,7 @@ only recommended for very secure environments.
       .*:
         - .*
 
-This configuration will allow minions with IDs ending in ``.example.com`` access
+This configuration allows minions with IDs ending in ``.example.com`` access
 to the test, ps, and pkg module functions.
 
 .. code-block:: yaml

From 5a10df14edb83e274dcfa5f33911dadb451f1154 Mon Sep 17 00:00:00 2001
From: Augustas 
Date: Mon, 31 May 2021 18:12:06 +0200
Subject: [PATCH 147/152] Update walkthrough_macosx.rst

precise64 image is non existing anymore on vagrantup servers. Also changed instructions with new approach to setup virtualbox image name while executing vagrant init command.
---
 doc/topics/tutorials/walkthrough_macosx.rst | 28 +++------------------
 1 file changed, 4 insertions(+), 24 deletions(-)

diff --git a/doc/topics/tutorials/walkthrough_macosx.rst b/doc/topics/tutorials/walkthrough_macosx.rst
index 40e29286196..3a966291a9b 100644
--- a/doc/topics/tutorials/walkthrough_macosx.rst
+++ b/doc/topics/tutorials/walkthrough_macosx.rst
@@ -246,36 +246,16 @@ From the minion folder, type
 
 .. code-block:: bash
 
-    vagrant init
+    vagrant init ubuntu/focal64
 
-This command creates a default Vagrantfile configuration file. This
+This command creates a default Vagrantfile configuration file and import focal64 virtualbox image file to configuration, so it could be used. This
 configuration file will be used to pass configuration parameters to the Salt
 provisioner in Step 3.
 
-Import Precise64 Ubuntu Box
----------------------------
-
-.. code-block:: bash
-
-    vagrant box add precise64 http://files.vagrantup.com/precise64.box
-
-.. note::
-
-    This box is added at the global Vagrant level. You only need to do it
-    once as each VM will use this same file.
-
 Modify the Vagrantfile
 ----------------------
 
-Modify ./minion/Vagrantfile to use th precise64 box. Change the ``config.vm.box``
-line to:
-
-.. code-block:: yaml
-
-    config.vm.box = "precise64"
-
-Uncomment the line creating a host-only IP. This is the ip of your minion
-(you can change it to something else if that IP is already in use):
+Modify Vagrantfile to use th private_ip in local network.
 
 .. code-block:: yaml
 
@@ -310,7 +290,7 @@ Now log into the VM in ssh using Vagrant again:
     vagrant ssh
 
 You should see the shell prompt change to something similar to
-``vagrant@precise64:~$`` meaning you're inside the VM. From there, enter the
+``vagrant@focal64:~$`` meaning you're inside the VM. From there, enter the
 following:
 
 .. code-block:: bash

From a89d53d89c2f580503960d5c7fd7ea76c2e1dac9 Mon Sep 17 00:00:00 2001
From: Elias Probst 
Date: Mon, 29 Jul 2019 07:56:57 +0000
Subject: [PATCH 148/152] auth: correct typos in docstrings

---
 salt/auth/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/auth/__init__.py b/salt/auth/__init__.py
index b87e2aff0df..65d2a1ae37d 100644
--- a/salt/auth/__init__.py
+++ b/salt/auth/__init__.py
@@ -267,7 +267,7 @@ class LoadAuth:
 
     def list_tokens(self):
         """
-        List all tokens in eauth_tokn storage.
+        List all tokens in eauth_tokens storage.
         """
         return self.tokens["{}.list_tokens".format(self.opts["eauth_tokens"])](
             self.opts
@@ -605,7 +605,7 @@ class AuthUser:
         """
         Instantiate an AuthUser object.
 
-        Takes a user to reprsent, as a string.
+        Takes a user to represent, as a string.
         """
         self.user = user
 

From 4b8fea8607e5e217d8fb776674bcc10157eaa8bb Mon Sep 17 00:00:00 2001
From: piterpunk 
Date: Thu, 30 Jul 2020 04:59:10 -0300
Subject: [PATCH 149/152] Added changelog entry

---
 changelog/54047.fixed | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog/54047.fixed

diff --git a/changelog/54047.fixed b/changelog/54047.fixed
new file mode 100644
index 00000000000..9852523f081
--- /dev/null
+++ b/changelog/54047.fixed
@@ -0,0 +1 @@
+Fixes typos in auth docstrings

From f012ab631394cfa23a2de2b9274345fb42bab14b Mon Sep 17 00:00:00 2001
From: Megan Wilhite 
Date: Wed, 31 May 2023 13:09:12 -0600
Subject: [PATCH 150/152] Remove changelog

---
 changelog/54047.fixed | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 changelog/54047.fixed

diff --git a/changelog/54047.fixed b/changelog/54047.fixed
deleted file mode 100644
index 9852523f081..00000000000
--- a/changelog/54047.fixed
+++ /dev/null
@@ -1 +0,0 @@
-Fixes typos in auth docstrings

From 098dae15cb2fea8b6bff98e30fa3b3013f2676f1 Mon Sep 17 00:00:00 2001
From: Daniel Mach 
Date: Thu, 18 May 2023 09:26:21 +0200
Subject: [PATCH 151/152] Migrate string formatting in 'pass' renderer to a
 f-string

---
 salt/renderers/pass.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/salt/renderers/pass.py b/salt/renderers/pass.py
index ba0f152c23e..17eafffa724 100644
--- a/salt/renderers/pass.py
+++ b/salt/renderers/pass.py
@@ -159,9 +159,7 @@ def _fetch_secret(pass_path):
             pass_error = pass_error.decode("utf-8")
         except (AttributeError, ValueError):
             pass
-        msg = "Could not fetch secret '{}' from the password store: {}".format(
-            pass_path, pass_error
-        )
+        msg = f"Could not fetch secret '{pass_path}' from the password store: {pass_error}"
         if pass_strict_fetch:
             raise SaltRenderError(msg)
         else:

From 8dfc923876e4a9b6e88efb0a5598c93dbbf967da Mon Sep 17 00:00:00 2001
From: Daniel Mach 
Date: Thu, 18 May 2023 10:15:03 +0200
Subject: [PATCH 152/152] Fix utf8 handling in 'pass' renderer and make it more
 robust

---
 changelog/64300.fixed.md                  |  1 +
 salt/renderers/pass.py                    |  8 +-
 tests/pytests/unit/renderers/test_pass.py | 99 +++++++++++++++++++++++
 3 files changed, 102 insertions(+), 6 deletions(-)
 create mode 100644 changelog/64300.fixed.md

diff --git a/changelog/64300.fixed.md b/changelog/64300.fixed.md
new file mode 100644
index 00000000000..4418db1d04c
--- /dev/null
+++ b/changelog/64300.fixed.md
@@ -0,0 +1 @@
+Fix utf8 handling in 'pass' renderer
diff --git a/salt/renderers/pass.py b/salt/renderers/pass.py
index 17eafffa724..ae75bba443b 100644
--- a/salt/renderers/pass.py
+++ b/salt/renderers/pass.py
@@ -145,20 +145,16 @@ def _fetch_secret(pass_path):
         env["GNUPGHOME"] = pass_gnupghome
 
     try:
-        proc = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env)
+        proc = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, encoding="utf-8")
         pass_data, pass_error = proc.communicate()
         pass_returncode = proc.returncode
-    except OSError as e:
+    except (OSError, UnicodeDecodeError) as e:
         pass_data, pass_error = "", str(e)
         pass_returncode = 1
 
     # The version of pass used during development sent output to
     # stdout instead of stderr even though its returncode was non zero.
     if pass_returncode or not pass_data:
-        try:
-            pass_error = pass_error.decode("utf-8")
-        except (AttributeError, ValueError):
-            pass
         msg = f"Could not fetch secret '{pass_path}' from the password store: {pass_error}"
         if pass_strict_fetch:
             raise SaltRenderError(msg)
diff --git a/tests/pytests/unit/renderers/test_pass.py b/tests/pytests/unit/renderers/test_pass.py
index 1e2ebb7ea8b..f7c79e1fe17 100644
--- a/tests/pytests/unit/renderers/test_pass.py
+++ b/tests/pytests/unit/renderers/test_pass.py
@@ -1,8 +1,12 @@
 import importlib
+import os
+import shutil
+import tempfile
 
 import pytest
 
 import salt.exceptions
+import salt.utils.files
 from tests.support.mock import MagicMock, patch
 
 # "pass" is a reserved keyword, we need to import it differently
@@ -19,6 +23,47 @@ def configure_loader_modules(master_opts):
     }
 
 
+@pytest.fixture()
+def pass_executable(request):
+    tmp_dir = tempfile.mkdtemp(prefix="salt_pass_")
+    pass_path = os.path.join(tmp_dir, "pass")
+    with salt.utils.files.fopen(pass_path, "w") as f:
+        f.write("#!/bin/sh\n")
+        # return path path wrapped into unicode characters
+        # pass args ($1, $2) are ("show", )
+        f.write('echo "α>>> $2 <<<β"\n')
+    os.chmod(pass_path, 0o755)
+    yield pass_path
+    shutil.rmtree(tmp_dir)
+
+
+@pytest.fixture()
+def pass_executable_error(request):
+    tmp_dir = tempfile.mkdtemp(prefix="salt_pass_")
+    pass_path = os.path.join(tmp_dir, "pass")
+    with salt.utils.files.fopen(pass_path, "w") as f:
+        f.write("#!/bin/sh\n")
+        # return error message with unicode characters
+        f.write('echo "ERROR: αβγ" >&2\n')
+        f.write("exit 1\n")
+    os.chmod(pass_path, 0o755)
+    yield pass_path
+    shutil.rmtree(tmp_dir)
+
+
+@pytest.fixture()
+def pass_executable_invalid_utf8(request):
+    tmp_dir = tempfile.mkdtemp(prefix="salt_pass_")
+    pass_path = os.path.join(tmp_dir, "pass")
+    with salt.utils.files.fopen(pass_path, "wb") as f:
+        f.write(b"#!/bin/sh\n")
+        # return invalid utf-8 sequence
+        f.write(b'echo "\x80\x81"\n')
+    os.chmod(pass_path, 0o755)
+    yield pass_path
+    shutil.rmtree(tmp_dir)
+
+
 # The default behavior is that if fetching a secret from pass fails,
 # the value is passed through. Even the trailing newlines are preserved.
 def test_passthrough():
@@ -161,3 +206,57 @@ def test_env():
     call_args, call_kwargs = popen_mock.call_args_list[0]
     assert call_kwargs["env"]["GNUPGHOME"] == config["pass_gnupghome"]
     assert call_kwargs["env"]["PASSWORD_STORE_DIR"] == config["pass_dir"]
+
+
+@pytest.mark.skip_on_windows(reason="Not supported on Windows")
+def test_utf8(pass_executable):
+    config = {
+        "pass_variable_prefix": "pass:",
+        "pass_strict_fetch": True,
+    }
+    mocks = {
+        "_get_pass_exec": MagicMock(return_value=pass_executable),
+    }
+
+    pass_path = "pass:secret"
+    with patch.dict(pass_.__opts__, config), patch.dict(pass_.__dict__, mocks):
+        result = pass_.render(pass_path)
+    assert result == "α>>> secret <<<β"
+
+
+@pytest.mark.skip_on_windows(reason="Not supported on Windows")
+def test_utf8_error(pass_executable_error):
+    config = {
+        "pass_variable_prefix": "pass:",
+        "pass_strict_fetch": True,
+    }
+    mocks = {
+        "_get_pass_exec": MagicMock(return_value=pass_executable_error),
+    }
+
+    pass_path = "pass:secret"
+    with patch.dict(pass_.__opts__, config), patch.dict(pass_.__dict__, mocks):
+        with pytest.raises(
+            salt.exceptions.SaltRenderError,
+            match=r"Could not fetch secret 'secret' from the password store: ERROR: αβγ",
+        ):
+            result = pass_.render(pass_path)
+
+
+@pytest.mark.skip_on_windows(reason="Not supported on Windows")
+def test_invalid_utf8(pass_executable_invalid_utf8):
+    config = {
+        "pass_variable_prefix": "pass:",
+        "pass_strict_fetch": True,
+    }
+    mocks = {
+        "_get_pass_exec": MagicMock(return_value=pass_executable_invalid_utf8),
+    }
+
+    pass_path = "pass:secret"
+    with patch.dict(pass_.__opts__, config), patch.dict(pass_.__dict__, mocks):
+        with pytest.raises(
+            salt.exceptions.SaltRenderError,
+            match=r"Could not fetch secret 'secret' from the password store: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte",
+        ):
+            result = pass_.render(pass_path)