From c89ca5b9f1f6fd730ffa347af9ff9bc6a76fb759 Mon Sep 17 00:00:00 2001 From: hurzhurz Date: Tue, 28 Nov 2023 13:11:02 +0100 Subject: [PATCH 01/29] s3fs: add settings for hardcoded parameters S3_SYNC_ON_UPDATE and S3_CACHE_EXPIRE --- salt/fileserver/s3fs.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/salt/fileserver/s3fs.py b/salt/fileserver/s3fs.py index ac1c8e41597..e4ad5e5c6b5 100644 --- a/salt/fileserver/s3fs.py +++ b/salt/fileserver/s3fs.py @@ -75,6 +75,16 @@ structure:: More info here: https://docs.aws.amazon.com/cli/latest/topic/s3-config.html + +.. note:: This fileserver back-end will by default sync all buckets on every + fileserver update. + + If you want files to be only populated in the cache when requested, you can + disable this in the master config: + + .. code-block:: yaml + + s3.s3_sync_on_update: False """ @@ -94,9 +104,6 @@ import salt.utils.versions log = logging.getLogger(__name__) -S3_CACHE_EXPIRE = 30 # cache for 30 seconds -S3_SYNC_ON_UPDATE = True # sync cache on update rather than jit - def envs(): """ @@ -116,7 +123,8 @@ def update(): metadata = _init() - if S3_SYNC_ON_UPDATE: + # sync cache on update rather than jit + if __opts__.get("s3.s3_sync_on_update", True): # sync the buckets to the local cache log.info("Syncing local cache from S3...") for saltenv, env_meta in metadata.items(): @@ -343,7 +351,7 @@ def _init(): specified and cache the data to disk. """ cache_file = _get_buckets_cache_filename() - exp = time.time() - S3_CACHE_EXPIRE + exp = time.time() - __opts__.get("s3.s3_cache_expire", 30) # check mtime of the buckets files cache metadata = None From e10482ecaa92992abe6b2156e27b00c9cff0fd82 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 20 Jan 2024 18:20:05 +0000 Subject: [PATCH 02/29] Fix some more places where we need to use ``actions/{upload,download}-artifact@v3`` Signed-off-by: Pedro Algarvio --- .github/actions/download-artifact/action.yml | 4 +++- .github/actions/upload-artifact/action.yml | 4 +++- .github/workflows/nightly.yml | 4 +++- .github/workflows/staging.yml | 4 +++- .github/workflows/templates/nightly.yml.jinja | 4 +++- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/actions/download-artifact/action.yml b/.github/actions/download-artifact/action.yml index fd476d5a8fd..12a097f517f 100644 --- a/.github/actions/download-artifact/action.yml +++ b/.github/actions/download-artifact/action.yml @@ -23,7 +23,9 @@ inputs: runs: using: composite steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v3 + # This needs to be actions/download-artifact@v3 because we upload multiple artifacts + # under the same name something that actions/upload-artifact@v4 does not do. with: name: ${{ inputs.name }} path: ${{ inputs.path }} diff --git a/.github/actions/upload-artifact/action.yml b/.github/actions/upload-artifact/action.yml index d1371d134f2..6c9e940a3b3 100644 --- a/.github/actions/upload-artifact/action.yml +++ b/.github/actions/upload-artifact/action.yml @@ -45,7 +45,9 @@ runs: shopt -s globstar || echo "'globstar' not available" tar -cavf ${{ inputs.archive-name || inputs.name || 'archive' }}.tar.gz ${{ inputs.path }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v3 + # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts + # under the same name something that actions/upload-artifact@v4 does not do. with: name: ${{ inputs.name }} path: ${{ inputs.archive-name || inputs.name || 'archive' }}.tar.gz diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 13f1ad225a8..276ff27c832 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2922,7 +2922,9 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - name: Download Repository Artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 + # This needs to be actions/download-artifact@v3 because we upload multiple artifacts + # under the same name something that actions/upload-artifact@v4 does not do. with: name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo path: repo/ diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index b8c9e0ce4c9..00b58b8ad6e 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -2732,7 +2732,9 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - name: Download Repository Artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 + # This needs to be actions/download-artifact@v3 because we upload multiple artifacts + # under the same name something that actions/upload-artifact@v4 does not do. with: name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo path: repo/ diff --git a/.github/workflows/templates/nightly.yml.jinja b/.github/workflows/templates/nightly.yml.jinja index 1487ab8ba26..097ccc7d2bb 100644 --- a/.github/workflows/templates/nightly.yml.jinja +++ b/.github/workflows/templates/nightly.yml.jinja @@ -170,7 +170,9 @@ concurrency: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - name: Download Repository Artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 + # This needs to be actions/download-artifact@v3 because we upload multiple artifacts + # under the same name something that actions/upload-artifact@v4 does not do. with: name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo path: repo/ From bfc1e9ad41c18469559031cc68d1828fdf3ad72d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 22 Jan 2024 08:42:26 +0000 Subject: [PATCH 03/29] Reduce the number of test swarm minions on Amazon 2023 on Arm64 The test suite is getting OOM killed on that test. Signed-off-by: Pedro Algarvio --- tests/pytests/scenarios/swarm/conftest.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/pytests/scenarios/swarm/conftest.py b/tests/pytests/scenarios/swarm/conftest.py index 8147eb8049d..c69ee10c674 100644 --- a/tests/pytests/scenarios/swarm/conftest.py +++ b/tests/pytests/scenarios/swarm/conftest.py @@ -39,19 +39,32 @@ def salt_cli(salt_master): @pytest.fixture(scope="package") -def minion_count(): +def _minion_count(grains): # Allow this to be changed via an environment variable if needed - return int(os.environ.get("SALT_CI_MINION_SWARM_COUNT", 15)) + env_count = os.environ.get("SALT_CI_MINION_SWARM_COUNT") + if env_count is not None: + return int(env_count) + # Default to 15 swarm minions + count = 15 + if grains["osarch"] != "osarch": + return count + if grains["os"] != "Amazon": + return count + if grains["osmajorrelease"] != 2023: + return count + # Looks like the test suite on Amazon 2023 under ARM64 get's OOM killed + # Let's reduce the number of swarm minions + return count - 5 @pytest.fixture(scope="package") -def minion_swarm(salt_master, minion_count): +def minion_swarm(salt_master, _minion_count): assert salt_master.is_running() minions = [] # We create and arbitrarily tall context stack to register the # minions stop mechanism callback with ExitStack() as stack: - for idx in range(minion_count): + for idx in range(_minion_count): minion_factory = salt_master.salt_minion_daemon( random_string(f"swarm-minion-{idx}-"), extra_cli_arguments_after_first_start_failure=["--log-level=info"], From 56f73ba939e7861b4a10e797f1fa43c1371b38ff Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 22 Jan 2024 17:32:39 +0000 Subject: [PATCH 04/29] Fix `osarch` value in comparisson Signed-off-by: Pedro Algarvio --- tests/pytests/scenarios/swarm/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytests/scenarios/swarm/conftest.py b/tests/pytests/scenarios/swarm/conftest.py index c69ee10c674..f1d9154e9e1 100644 --- a/tests/pytests/scenarios/swarm/conftest.py +++ b/tests/pytests/scenarios/swarm/conftest.py @@ -46,7 +46,7 @@ def _minion_count(grains): return int(env_count) # Default to 15 swarm minions count = 15 - if grains["osarch"] != "osarch": + if grains["osarch"] != "aarch64": return count if grains["os"] != "Amazon": return count From c91e1d8a5edea7e1025b9b95b63b05410578b42a Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Thu, 18 Jan 2024 02:23:37 +0000 Subject: [PATCH 05/29] add https to proxy so that requests knows how to proxy https --- changelog/65824.fixed.md | 1 + requirements/pytest.txt | 1 + requirements/static/ci/py3.10/cloud.txt | 6 +++ requirements/static/ci/py3.10/darwin.txt | 4 ++ requirements/static/ci/py3.10/freebsd.txt | 4 ++ requirements/static/ci/py3.10/linux.txt | 4 ++ requirements/static/ci/py3.10/windows.txt | 4 ++ requirements/static/ci/py3.11/cloud.txt | 6 +++ requirements/static/ci/py3.11/darwin.txt | 4 ++ requirements/static/ci/py3.11/freebsd.txt | 4 ++ requirements/static/ci/py3.11/linux.txt | 4 ++ requirements/static/ci/py3.11/windows.txt | 4 ++ requirements/static/ci/py3.12/cloud.txt | 6 +++ requirements/static/ci/py3.12/darwin.txt | 4 ++ requirements/static/ci/py3.12/freebsd.txt | 4 ++ requirements/static/ci/py3.12/linux.txt | 4 ++ requirements/static/ci/py3.12/windows.txt | 4 ++ requirements/static/ci/py3.7/cloud.txt | 6 +++ requirements/static/ci/py3.7/freebsd.txt | 4 ++ requirements/static/ci/py3.7/linux.txt | 4 ++ requirements/static/ci/py3.7/windows.txt | 4 ++ requirements/static/ci/py3.8/cloud.txt | 6 +++ requirements/static/ci/py3.8/freebsd.txt | 4 ++ requirements/static/ci/py3.8/linux.txt | 4 ++ requirements/static/ci/py3.8/windows.txt | 4 ++ requirements/static/ci/py3.9/cloud.txt | 6 +++ requirements/static/ci/py3.9/darwin.txt | 4 ++ requirements/static/ci/py3.9/freebsd.txt | 4 ++ requirements/static/ci/py3.9/linux.txt | 4 ++ requirements/static/ci/py3.9/windows.txt | 4 ++ salt/utils/http.py | 5 ++- tests/pytests/functional/utils/test_http.py | 41 ++++++++++++++++----- 32 files changed, 162 insertions(+), 10 deletions(-) create mode 100644 changelog/65824.fixed.md diff --git a/changelog/65824.fixed.md b/changelog/65824.fixed.md new file mode 100644 index 00000000000..213f3f505fb --- /dev/null +++ b/changelog/65824.fixed.md @@ -0,0 +1 @@ +added https proxy to the list of proxies so that requests knows what to do with https based proxies diff --git a/requirements/pytest.txt b/requirements/pytest.txt index 209db530ab1..23e83b8092c 100644 --- a/requirements/pytest.txt +++ b/requirements/pytest.txt @@ -11,3 +11,4 @@ pytest-custom-exit-code >= 0.3 flaky more-itertools pyfakefs +trustme diff --git a/requirements/static/ci/py3.10/cloud.txt b/requirements/static/ci/py3.10/cloud.txt index 3460caf2d22..b891ab132a1 100644 --- a/requirements/static/ci/py3.10/cloud.txt +++ b/requirements/static/ci/py3.10/cloud.txt @@ -135,6 +135,7 @@ cryptography==41.0.7 # pyspnego # requests-ntlm # smbprotocol + # trustme # vcert distlib==0.3.2 # via @@ -202,6 +203,7 @@ idna==3.2 # -c requirements/static/ci/py3.10/linux.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -670,6 +672,10 @@ transitions==0.8.9 # via # -c requirements/static/ci/py3.10/linux.txt # junos-eznc +trustme==1.1.0 + # via + # -c requirements/static/ci/py3.10/linux.txt + # -r requirements/pytest.txt typing-extensions==4.8.0 # via # -c requirements/static/ci/py3.10/linux.txt diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index a2a7e8fc505..632dd4d7391 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -97,6 +97,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -146,6 +147,7 @@ idna==3.2 # -r requirements/darwin.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -463,6 +465,8 @@ tomli==2.0.1 # via pytest transitions==0.8.9 # via junos-eznc +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.2.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index 8a9235c5beb..198053f59da 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -94,6 +94,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -138,6 +139,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.10/freebsd.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -455,6 +457,8 @@ tomli==2.0.1 # via pytest transitions==0.8.9 # via junos-eznc +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.8.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index fdafc6c5ac2..b017f0a3ee1 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -104,6 +104,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -147,6 +148,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.10/linux.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -500,6 +502,8 @@ tornado==6.1 # via python-telegram-bot transitions==0.8.9 # via junos-eznc +trustme==1.1.0 + # via -r requirements/pytest.txt twilio==7.9.2 # via -r requirements/static/ci/linux.in typing-extensions==4.8.0 diff --git a/requirements/static/ci/py3.10/windows.txt b/requirements/static/ci/py3.10/windows.txt index a9138960e18..05dcf33f303 100644 --- a/requirements/static/ci/py3.10/windows.txt +++ b/requirements/static/ci/py3.10/windows.txt @@ -89,6 +89,7 @@ cryptography==41.0.7 # moto # pyopenssl # requests-ntlm + # trustme distlib==0.3.6 # via virtualenv distro==1.5.0 @@ -136,6 +137,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.10/windows.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -446,6 +448,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.4.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.11/cloud.txt b/requirements/static/ci/py3.11/cloud.txt index 441d621bccf..d2d4afcc69d 100644 --- a/requirements/static/ci/py3.11/cloud.txt +++ b/requirements/static/ci/py3.11/cloud.txt @@ -131,6 +131,7 @@ cryptography==41.0.7 # pyspnego # requests-ntlm # smbprotocol + # trustme # vcert distlib==0.3.2 # via @@ -194,6 +195,7 @@ idna==3.2 # -c requirements/static/ci/py3.11/linux.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -617,6 +619,10 @@ toml==0.10.2 # via # -c requirements/static/ci/py3.11/linux.txt # -r requirements/static/ci/common.in +trustme==1.1.0 + # via + # -c requirements/static/ci/py3.11/linux.txt + # -r requirements/pytest.txt typing-extensions==4.8.0 # via # -c requirements/static/ci/py3.11/linux.txt diff --git a/requirements/static/ci/py3.11/darwin.txt b/requirements/static/ci/py3.11/darwin.txt index 6b3a176d31a..c8f787d31e5 100644 --- a/requirements/static/ci/py3.11/darwin.txt +++ b/requirements/static/ci/py3.11/darwin.txt @@ -92,6 +92,7 @@ cryptography==41.0.7 # etcd3-py # moto # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -139,6 +140,7 @@ idna==3.2 # -r requirements/darwin.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -424,6 +426,8 @@ timelib==0.2.5 # -r requirements/darwin.txt toml==0.10.2 # via -r requirements/static/ci/common.in +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.2.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.11/freebsd.txt b/requirements/static/ci/py3.11/freebsd.txt index 5ebd4c09d73..34cd70a2d48 100644 --- a/requirements/static/ci/py3.11/freebsd.txt +++ b/requirements/static/ci/py3.11/freebsd.txt @@ -92,6 +92,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -134,6 +135,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.11/freebsd.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -421,6 +423,8 @@ timelib==0.2.5 # -r requirements/static/pkg/freebsd.in toml==0.10.2 # via -r requirements/static/ci/common.in +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.8.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.11/linux.txt b/requirements/static/ci/py3.11/linux.txt index 37dc2c11968..0869197b905 100644 --- a/requirements/static/ci/py3.11/linux.txt +++ b/requirements/static/ci/py3.11/linux.txt @@ -102,6 +102,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -143,6 +144,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.11/linux.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -466,6 +468,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tornado==6.1 # via python-telegram-bot +trustme==1.1.0 + # via -r requirements/pytest.txt twilio==7.9.2 # via -r requirements/static/ci/linux.in typing-extensions==4.8.0 diff --git a/requirements/static/ci/py3.11/windows.txt b/requirements/static/ci/py3.11/windows.txt index 4b2a5d363d7..358ff340067 100644 --- a/requirements/static/ci/py3.11/windows.txt +++ b/requirements/static/ci/py3.11/windows.txt @@ -87,6 +87,7 @@ cryptography==41.0.7 # moto # pyopenssl # requests-ntlm + # trustme distlib==0.3.6 # via virtualenv distro==1.5.0 @@ -132,6 +133,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.11/windows.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -440,6 +442,8 @@ timelib==0.2.5 # -r requirements/windows.txt toml==0.10.2 # via -r requirements/static/ci/common.in +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.4.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.12/cloud.txt b/requirements/static/ci/py3.12/cloud.txt index 787b2855ed5..7d05aded66a 100644 --- a/requirements/static/ci/py3.12/cloud.txt +++ b/requirements/static/ci/py3.12/cloud.txt @@ -131,6 +131,7 @@ cryptography==41.0.7 # pyspnego # requests-ntlm # smbprotocol + # trustme # vcert distlib==0.3.2 # via @@ -194,6 +195,7 @@ idna==3.2 # -c requirements/static/ci/py3.12/linux.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -617,6 +619,10 @@ toml==0.10.2 # via # -c requirements/static/ci/py3.12/linux.txt # -r requirements/static/ci/common.in +trustme==1.1.0 + # via + # -c requirements/static/ci/py3.12/linux.txt + # -r requirements/pytest.txt typing-extensions==4.8.0 # via # -c requirements/static/ci/py3.12/linux.txt diff --git a/requirements/static/ci/py3.12/darwin.txt b/requirements/static/ci/py3.12/darwin.txt index 13902e31d8c..770ac1d7902 100644 --- a/requirements/static/ci/py3.12/darwin.txt +++ b/requirements/static/ci/py3.12/darwin.txt @@ -92,6 +92,7 @@ cryptography==41.0.7 # etcd3-py # moto # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -139,6 +140,7 @@ idna==3.2 # -r requirements/darwin.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -424,6 +426,8 @@ timelib==0.2.5 # -r requirements/darwin.txt toml==0.10.2 # via -r requirements/static/ci/common.in +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.2.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.12/freebsd.txt b/requirements/static/ci/py3.12/freebsd.txt index 88260a34989..9c3e9e51ddb 100644 --- a/requirements/static/ci/py3.12/freebsd.txt +++ b/requirements/static/ci/py3.12/freebsd.txt @@ -92,6 +92,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -134,6 +135,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.12/freebsd.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -421,6 +423,8 @@ timelib==0.2.5 # -r requirements/static/pkg/freebsd.in toml==0.10.2 # via -r requirements/static/ci/common.in +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.8.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.12/linux.txt b/requirements/static/ci/py3.12/linux.txt index 1d7d02b82b7..7c64e8eac5b 100644 --- a/requirements/static/ci/py3.12/linux.txt +++ b/requirements/static/ci/py3.12/linux.txt @@ -102,6 +102,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -143,6 +144,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.12/linux.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -466,6 +468,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tornado==6.1 # via python-telegram-bot +trustme==1.1.0 + # via -r requirements/pytest.txt twilio==7.9.2 # via -r requirements/static/ci/linux.in typing-extensions==4.8.0 diff --git a/requirements/static/ci/py3.12/windows.txt b/requirements/static/ci/py3.12/windows.txt index f0109605d12..d943b70255b 100644 --- a/requirements/static/ci/py3.12/windows.txt +++ b/requirements/static/ci/py3.12/windows.txt @@ -87,6 +87,7 @@ cryptography==41.0.7 # moto # pyopenssl # requests-ntlm + # trustme distlib==0.3.6 # via virtualenv distro==1.5.0 @@ -132,6 +133,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.12/windows.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -440,6 +442,8 @@ timelib==0.2.5 # -r requirements/windows.txt toml==0.10.2 # via -r requirements/static/ci/common.in +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.4.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.7/cloud.txt b/requirements/static/ci/py3.7/cloud.txt index 36af32798f6..81efd207812 100644 --- a/requirements/static/ci/py3.7/cloud.txt +++ b/requirements/static/ci/py3.7/cloud.txt @@ -149,6 +149,7 @@ cryptography==41.0.7 # pyspnego # requests-ntlm # smbprotocol + # trustme # vcert distlib==0.3.2 # via @@ -222,6 +223,7 @@ idna==3.2 # -c requirements/static/ci/py3.7/linux.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -736,6 +738,10 @@ transitions==0.8.9 # via # -c requirements/static/ci/py3.7/linux.txt # junos-eznc +trustme==1.0.0 + # via + # -c requirements/static/ci/py3.7/linux.txt + # -r requirements/pytest.txt typing-extensions==3.10.0.0 # via # -c requirements/static/ci/../pkg/py3.7/linux.txt diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt index f54e45169ee..f3171516c07 100644 --- a/requirements/static/ci/py3.7/freebsd.txt +++ b/requirements/static/ci/py3.7/freebsd.txt @@ -104,6 +104,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -153,6 +154,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.7/freebsd.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -510,6 +512,8 @@ tomli==2.0.1 # via pytest transitions==0.8.9 # via junos-eznc +trustme==1.0.0 + # via -r requirements/pytest.txt typing-extensions==3.10.0.0 # via # -c requirements/static/ci/../pkg/py3.7/freebsd.txt diff --git a/requirements/static/ci/py3.7/linux.txt b/requirements/static/ci/py3.7/linux.txt index 2dd4e146ee1..36708ecfa88 100644 --- a/requirements/static/ci/py3.7/linux.txt +++ b/requirements/static/ci/py3.7/linux.txt @@ -111,6 +111,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -159,6 +160,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.7/linux.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -547,6 +549,8 @@ tornado==6.1 # via python-telegram-bot transitions==0.8.9 # via junos-eznc +trustme==1.0.0 + # via -r requirements/pytest.txt twilio==7.9.2 # via -r requirements/static/ci/linux.in typing-extensions==3.10.0.0 diff --git a/requirements/static/ci/py3.7/windows.txt b/requirements/static/ci/py3.7/windows.txt index 18670294d1d..c942618b2ce 100644 --- a/requirements/static/ci/py3.7/windows.txt +++ b/requirements/static/ci/py3.7/windows.txt @@ -96,6 +96,7 @@ cryptography==41.0.7 # moto # pyopenssl # requests-ntlm + # trustme distlib==0.3.2 # via virtualenv distro==1.5.0 @@ -143,6 +144,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.7/windows.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -462,6 +464,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +trustme==1.0.0 + # via -r requirements/pytest.txt typing-extensions==4.4.0 # via # -c requirements/static/ci/../pkg/py3.7/windows.txt diff --git a/requirements/static/ci/py3.8/cloud.txt b/requirements/static/ci/py3.8/cloud.txt index e6262cb5178..977351155da 100644 --- a/requirements/static/ci/py3.8/cloud.txt +++ b/requirements/static/ci/py3.8/cloud.txt @@ -144,6 +144,7 @@ cryptography==41.0.7 # pyspnego # requests-ntlm # smbprotocol + # trustme # vcert distlib==0.3.2 # via @@ -217,6 +218,7 @@ idna==3.2 # -c requirements/static/ci/py3.8/linux.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -723,6 +725,10 @@ transitions==0.8.9 # via # -c requirements/static/ci/py3.8/linux.txt # junos-eznc +trustme==1.1.0 + # via + # -c requirements/static/ci/py3.8/linux.txt + # -r requirements/pytest.txt typing-extensions==4.8.0 # via # -c requirements/static/ci/py3.8/linux.txt diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index 6dff1b25d49..777b0b4eb7c 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -99,6 +99,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -148,6 +149,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.8/freebsd.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -497,6 +499,8 @@ tomli==2.0.1 # via pytest transitions==0.8.9 # via junos-eznc +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.8.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.8/linux.txt b/requirements/static/ci/py3.8/linux.txt index 7d0212b6958..a71925f5074 100644 --- a/requirements/static/ci/py3.8/linux.txt +++ b/requirements/static/ci/py3.8/linux.txt @@ -106,6 +106,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -154,6 +155,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.8/linux.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -534,6 +536,8 @@ tornado==6.1 # via python-telegram-bot transitions==0.8.9 # via junos-eznc +trustme==1.1.0 + # via -r requirements/pytest.txt twilio==7.9.2 # via -r requirements/static/ci/linux.in typing-extensions==4.8.0 diff --git a/requirements/static/ci/py3.8/windows.txt b/requirements/static/ci/py3.8/windows.txt index b9960dc20f7..02ae00bd118 100644 --- a/requirements/static/ci/py3.8/windows.txt +++ b/requirements/static/ci/py3.8/windows.txt @@ -91,6 +91,7 @@ cryptography==41.0.7 # moto # pyopenssl # requests-ntlm + # trustme distlib==0.3.2 # via virtualenv distro==1.5.0 @@ -138,6 +139,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.8/windows.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -449,6 +451,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.2.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.9/cloud.txt b/requirements/static/ci/py3.9/cloud.txt index ffdd686693e..6d2b7726781 100644 --- a/requirements/static/ci/py3.9/cloud.txt +++ b/requirements/static/ci/py3.9/cloud.txt @@ -144,6 +144,7 @@ cryptography==41.0.7 # pyspnego # requests-ntlm # smbprotocol + # trustme # vcert distlib==0.3.2 # via @@ -217,6 +218,7 @@ idna==3.2 # -c requirements/static/ci/py3.9/linux.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -725,6 +727,10 @@ transitions==0.8.9 # via # -c requirements/static/ci/py3.9/linux.txt # junos-eznc +trustme==1.1.0 + # via + # -c requirements/static/ci/py3.9/linux.txt + # -r requirements/pytest.txt typing-extensions==4.8.0 # via # -c requirements/static/ci/py3.9/linux.txt diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index c1a96550047..6c8e1fa6327 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -102,6 +102,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -156,6 +157,7 @@ idna==3.2 # -r requirements/darwin.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -507,6 +509,8 @@ tomli==2.0.1 # via pytest transitions==0.8.9 # via junos-eznc +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.2.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index 7133c4d45ed..a05648451a1 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -99,6 +99,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -148,6 +149,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.9/freebsd.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -499,6 +501,8 @@ tomli==2.0.1 # via pytest transitions==0.8.9 # via junos-eznc +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.8.0 # via # pytest-shell-utilities diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index d2b25cec1be..911b3457437 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -104,6 +104,7 @@ cryptography==41.0.7 # moto # paramiko # pyopenssl + # trustme # vcert distlib==0.3.2 # via virtualenv @@ -152,6 +153,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.9/linux.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -534,6 +536,8 @@ tornado==6.1 # via python-telegram-bot transitions==0.8.9 # via junos-eznc +trustme==1.1.0 + # via -r requirements/pytest.txt twilio==7.9.2 # via -r requirements/static/ci/linux.in typing-extensions==4.8.0 diff --git a/requirements/static/ci/py3.9/windows.txt b/requirements/static/ci/py3.9/windows.txt index 2bd40253c82..4c1e6281536 100644 --- a/requirements/static/ci/py3.9/windows.txt +++ b/requirements/static/ci/py3.9/windows.txt @@ -91,6 +91,7 @@ cryptography==41.0.7 # moto # pyopenssl # requests-ntlm + # trustme distlib==0.3.2 # via virtualenv distro==1.5.0 @@ -138,6 +139,7 @@ idna==3.2 # -c requirements/static/ci/../pkg/py3.9/windows.txt # etcd3-py # requests + # trustme # yarl immutables==0.15 # via @@ -450,6 +452,8 @@ toml==0.10.2 # via -r requirements/static/ci/common.in tomli==2.0.1 # via pytest +trustme==1.1.0 + # via -r requirements/pytest.txt typing-extensions==4.2.0 # via # pytest-shell-utilities diff --git a/salt/utils/http.py b/salt/utils/http.py index 5fae89efc8c..30c7fbdc8de 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -361,7 +361,10 @@ def query( sess_cookies = sess.cookies sess.verify = verify_ssl if http_proxy_url is not None: - sess.proxies = {"http": http_proxy_url} + sess.proxies = { + "http": http_proxy_url, + "https": http_proxy_url, + } elif backend == "urllib2": sess_cookies = None else: diff --git a/tests/pytests/functional/utils/test_http.py b/tests/pytests/functional/utils/test_http.py index e6e48d60570..874ef85eb48 100644 --- a/tests/pytests/functional/utils/test_http.py +++ b/tests/pytests/functional/utils/test_http.py @@ -1,7 +1,9 @@ import shutil +import ssl import tarfile import pytest +import trustme from pytestshellutils.utils import ports from saltfactories.utils import random_string @@ -40,6 +42,25 @@ def tinyproxy_pass(): return random_string("tinyproxy-pass-") +@pytest.fixture(scope="session") +def ca(): + return trustme.CA() + + +@pytest.fixture(scope="session") +def httpserver_ssl_context(ca): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + localhost_cert = ca.issue_cert("127.0.0.1") + localhost_cert.configure_cert(context) + return context + + +@pytest.fixture(scope="session") +def httpclient_ssl_context(ca): + with ca.cert_pem.tempfile() as ca_temp_path: + return ssl.create_default_context(cafile=ca_temp_path) + + @pytest.fixture(params=[True, False], ids=lambda x: "basic-auth" if x else "no-auth") def tinyproxy_basic_auth(request): return request.param @@ -107,6 +128,7 @@ def tinyproxy_container( def test_real_proxy( tinyproxy_container, httpserver, + ca, tinyproxy_port, tinyproxy_user, tinyproxy_pass, @@ -137,18 +159,19 @@ def test_real_proxy( else: httpserver.expect_request( "/real_proxy_test", - headers={"X-Tinyproxy-Header": "Test custom tinyproxy header"}, ).respond_with_data(data, content_type="application/octet-stream") url = httpserver.url_for("/real_proxy_test").replace("localhost", "127.0.0.1") # We just want to be sure that it's using the proxy - ret = salt.utils.http.query( - url, - method=http_method, - data=data, - backend=backend, - opts=opts, - decode_body=False, - ) + with ca.cert_pem.tempfile() as ca_temp_path: + ret = salt.utils.http.query( + url, + method=http_method, + data=data, + backend=backend, + opts=opts, + decode_body=False, + verify_ssl=ca_temp_path, + ) body = ret.get("body", "") assert body == data From 482ff737e66e6f725b116faa3c9f5b6af60babd6 Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Thu, 18 Jan 2024 03:07:02 +0000 Subject: [PATCH 06/29] try gate trustme --- tests/pytests/functional/utils/test_http.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/pytests/functional/utils/test_http.py b/tests/pytests/functional/utils/test_http.py index 874ef85eb48..f45b4409b31 100644 --- a/tests/pytests/functional/utils/test_http.py +++ b/tests/pytests/functional/utils/test_http.py @@ -3,7 +3,12 @@ import ssl import tarfile import pytest -import trustme + +try: + import trustme +except ImportError: + pass + from pytestshellutils.utils import ports from saltfactories.utils import random_string From d922f94e9552be517a65ca29f48bddb05074a26a Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Thu, 18 Jan 2024 05:57:37 +0000 Subject: [PATCH 07/29] fix http unit test --- tests/pytests/unit/utils/test_http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytests/unit/utils/test_http.py b/tests/pytests/unit/utils/test_http.py index dfb60085de2..91c4066b4da 100644 --- a/tests/pytests/unit/utils/test_http.py +++ b/tests/pytests/unit/utils/test_http.py @@ -245,7 +245,8 @@ def test_query_proxy(httpserver): ) assert mock_session.return_value.proxies == { - "http": "http://salt_test:super_secret@127.0.0.1:88" + "http": "http://salt_test:super_secret@127.0.0.1:88", + "https": "http://salt_test:super_secret@127.0.0.1:88", } opts["no_proxy"] = [httpserver.host] From ba44de34fa6fe82991966ccdc11fab26677dc975 Mon Sep 17 00:00:00 2001 From: Shane Lee Date: Wed, 10 Jan 2024 15:34:39 -0700 Subject: [PATCH 08/29] Rebase to 3006.x --- changelog/65777.fixed.md | 1 + salt/__init__.py | 50 ++---------------------------- salt/states/reg.py | 38 +++++++++++------------ salt/utils/win_reg.py | 41 ++++++++++++------------ tests/pytests/unit/test__init__.py | 6 ++++ 5 files changed, 49 insertions(+), 87 deletions(-) create mode 100644 changelog/65777.fixed.md create mode 100644 tests/pytests/unit/test__init__.py diff --git a/changelog/65777.fixed.md b/changelog/65777.fixed.md new file mode 100644 index 00000000000..3427b468878 --- /dev/null +++ b/changelog/65777.fixed.md @@ -0,0 +1 @@ +Fixes an issue when reading/modifying ini files that contain unicode characters diff --git a/salt/__init__.py b/salt/__init__.py index a7c32e159a4..2553ed40e9e 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -131,60 +131,16 @@ warnings.filterwarnings( def __define_global_system_encoding_variable__(): - import sys - - # This is the most trustworthy source of the system encoding, though, if - # salt is being imported after being daemonized, this information is lost - # and reset to None - encoding = None - - if not sys.platform.startswith("win") and sys.stdin is not None: - # On linux we can rely on sys.stdin for the encoding since it - # most commonly matches the filesystem encoding. This however - # does not apply to windows - encoding = sys.stdin.encoding - - if not encoding: - # If the system is properly configured this should return a valid - # encoding. MS Windows has problems with this and reports the wrong - # encoding - import locale - - try: - encoding = locale.getencoding() - except AttributeError: - # Python < 3.11 - encoding = locale.getpreferredencoding(do_setlocale=True) - - # This is now garbage collectable - del locale - - if not encoding: - # This is most likely ascii which is not the best but we were - # unable to find a better encoding. If this fails, we fall all - # the way back to ascii - encoding = sys.getdefaultencoding() - if not encoding: - if sys.platform.startswith("darwin"): - # Mac OS X uses UTF-8 - encoding = "utf-8" - elif sys.platform.startswith("win"): - # Windows uses a configurable encoding; on Windows, Python uses the name “mbcs” - # to refer to whatever the currently configured encoding is. - encoding = "mbcs" - else: - # On linux default to ascii as a last resort - encoding = "ascii" import builtins + import sys # Define the detected encoding as a built-in variable for ease of use - setattr(builtins, "__salt_system_encoding__", encoding) + setattr(builtins, "__salt_system_encoding__", sys.getdefaultencoding()) # This is now garbage collectable - del sys del builtins - del encoding + del sys __define_global_system_encoding_variable__() diff --git a/salt/states/reg.py b/salt/states/reg.py index 97359c01d84..964b01fa789 100644 --- a/salt/states/reg.py +++ b/salt/states/reg.py @@ -12,7 +12,7 @@ Hives This is the top level of the registry. They all begin with HKEY. - HKEY_CLASSES_ROOT (HKCR) - - HKEY_CURRENT_USER(HKCU) + - HKEY_CURRENT_USER (HKCU) - HKEY_LOCAL MACHINE (HKLM) - HKEY_USER (HKU) - HKEY_CURRENT_CONFIG @@ -139,7 +139,7 @@ def present( A string value representing the full path of the key to include the HIVE, Key, and all Subkeys. For example: - ``HKEY_LOCAL_MACHINE\\SOFTWARE\\Salt`` + ``HKEY_LOCAL_MACHINE\SOFTWARE\Salt`` Valid hive values include: @@ -298,24 +298,24 @@ def present( Example: The following example will set the ``(Default)`` value for the - ``SOFTWARE\\Salt`` key in the ``HKEY_CURRENT_USER`` hive to + ``SOFTWARE\Salt`` key in the ``HKEY_CURRENT_USER`` hive to ``2016.3.1``: .. code-block:: yaml - HKEY_CURRENT_USER\\SOFTWARE\\Salt: + HKEY_CURRENT_USER\SOFTWARE\Salt: reg.present: - vdata: 2016.3.1 Example: The following example will set the value for the ``version`` entry under - the ``SOFTWARE\\Salt`` key in the ``HKEY_CURRENT_USER`` hive to + the ``SOFTWARE\Salt`` key in the ``HKEY_CURRENT_USER`` hive to ``2016.3.1``. The value will be reflected in ``Wow6432Node``: .. code-block:: yaml - HKEY_CURRENT_USER\\SOFTWARE\\Salt: + HKEY_CURRENT_USER\SOFTWARE\Salt: reg.present: - vname: version - vdata: 2016.3.1 @@ -323,7 +323,7 @@ def present( In the above example the path is interpreted as follows: - ``HKEY_CURRENT_USER`` is the hive - - ``SOFTWARE\\Salt`` is the key + - ``SOFTWARE\Salt`` is the key - ``vname`` is the value name ('version') that will be created under the key - ``vdata`` is the data that will be assigned to 'version' @@ -423,7 +423,7 @@ def present( ) add_change = { - "Key": r"{}\{}".format(hive, key), + "Key": rf"{hive}\{key}", "Entry": "{}".format( salt.utils.stringutils.to_unicode(vname, "utf-8") if vname else "(Default)" ), @@ -451,10 +451,10 @@ def present( if not ret["result"]: ret["changes"] = {} - ret["comment"] = r"Failed to add {} to {}\{}".format(vname, hive, key) + ret["comment"] = rf"Failed to add {vname} to {hive}\{key}" else: ret["changes"] = {"reg": {"Added": add_change}} - ret["comment"] = r"Added {} to {}\{}".format(vname, hive, key) + ret["comment"] = rf"Added {vname} to {hive}\{key}" if ret["result"]: ret = __utils__["dacl.check_perms"]( @@ -480,7 +480,7 @@ def absent(name, vname=None, use_32bit_registry=False): A string value representing the full path of the key to include the HIVE, Key, and all Subkeys. For example: - ``HKEY_LOCAL_MACHINE\\SOFTWARE\\Salt`` + ``HKEY_LOCAL_MACHINE\SOFTWARE\Salt`` Valid hive values include: @@ -504,7 +504,7 @@ def absent(name, vname=None, use_32bit_registry=False): .. code-block:: yaml - 'HKEY_CURRENT_USER\\SOFTWARE\\Salt': + 'HKEY_CURRENT_USER\SOFTWARE\Salt': reg.absent - vname: version @@ -521,11 +521,11 @@ def absent(name, vname=None, use_32bit_registry=False): hive=hive, key=key, vname=vname, use_32bit_registry=use_32bit_registry ) if not reg_check["success"] or reg_check["vdata"] == "(value not set)": - ret["comment"] = "{} is already absent".format(name) + ret["comment"] = f"{name} is already absent" return ret remove_change = { - "Key": r"{}\{}".format(hive, key), + "Key": rf"{hive}\{key}", "Entry": "{}".format(vname if vname else "(Default)"), } @@ -541,10 +541,10 @@ def absent(name, vname=None, use_32bit_registry=False): ) if not ret["result"]: ret["changes"] = {} - ret["comment"] = r"Failed to remove {} from {}".format(key, hive) + ret["comment"] = rf"Failed to remove {key} from {hive}" else: ret["changes"] = {"reg": {"Removed": remove_change}} - ret["comment"] = r"Removed {} from {}".format(key, hive) + ret["comment"] = rf"Removed {key} from {hive}" return ret @@ -598,10 +598,10 @@ def key_absent(name, use_32bit_registry=False): if not __utils__["reg.read_value"]( hive=hive, key=key, use_32bit_registry=use_32bit_registry )["success"]: - ret["comment"] = "{} is already absent".format(name) + ret["comment"] = f"{name} is already absent" return ret - ret["changes"] = {"reg": {"Removed": {"Key": r"{}\{}".format(hive, key)}}} + ret["changes"] = {"reg": {"Removed": {"Key": rf"{hive}\{key}"}}} # Check for test option if __opts__["test"]: @@ -617,6 +617,6 @@ def key_absent(name, use_32bit_registry=False): )["success"]: ret["result"] = False ret["changes"] = {} - ret["comment"] = "Failed to remove registry key {}".format(name) + ret["comment"] = f"Failed to remove registry key {name}" return ret diff --git a/salt/utils/win_reg.py b/salt/utils/win_reg.py index 74aa17b5d81..d7063a86ad5 100644 --- a/salt/utils/win_reg.py +++ b/salt/utils/win_reg.py @@ -23,9 +23,8 @@ Values/Entries are name/data pairs. There can be many values in a key. The (Default) value corresponds to the Key itself, the rest are their own name/value pairs. -:depends: - PyWin32 +:depends: PyWin32 """ -# When production windows installer is using Python 3, Python 2 code can be removed import logging @@ -73,7 +72,7 @@ def __virtual__(): def _to_mbcs(vdata): """ - Converts unicode to to current users character encoding. Use this for values + Converts unicode to current users character encoding. Use this for values returned by reg functions """ return salt.utils.stringutils.to_unicode(vdata, "mbcs") @@ -186,7 +185,7 @@ def key_exists(hive, key, use_32bit_registry=False): try: hkey = registry.hkeys[local_hive] except KeyError: - raise CommandExecutionError("Invalid Hive: {}".format(local_hive)) + raise CommandExecutionError(f"Invalid Hive: {local_hive}") access_mask = registry.registry_32[use_32bit_registry] handle = None @@ -241,7 +240,7 @@ def value_exists(hive, key, vname, use_32bit_registry=False): try: hkey = registry.hkeys[local_hive] except KeyError: - raise CommandExecutionError("Invalid Hive: {}".format(local_hive)) + raise CommandExecutionError(f"Invalid Hive: {local_hive}") access_mask = registry.registry_32[use_32bit_registry] try: @@ -340,7 +339,7 @@ def list_keys(hive, key=None, use_32bit_registry=False): try: hkey = registry.hkeys[local_hive] except KeyError: - raise CommandExecutionError("Invalid Hive: {}".format(local_hive)) + raise CommandExecutionError(f"Invalid Hive: {local_hive}") access_mask = registry.registry_32[use_32bit_registry] subkeys = [] @@ -355,10 +354,10 @@ def list_keys(hive, key=None, use_32bit_registry=False): except win32api.error as exc: if exc.winerror == 2: log.debug(r"Cannot find key: %s\%s", hive, key, exc_info=True) - return False, r"Cannot find key: {}\{}".format(hive, key) + return False, rf"Cannot find key: {hive}\{key}" if exc.winerror == 5: log.debug(r"Access is denied: %s\%s", hive, key, exc_info=True) - return False, r"Access is denied: {}\{}".format(hive, key) + return False, rf"Access is denied: {hive}\{key}" raise finally: @@ -412,7 +411,7 @@ def list_values(hive, key=None, use_32bit_registry=False): try: hkey = registry.hkeys[local_hive] except KeyError: - raise CommandExecutionError("Invalid Hive: {}".format(local_hive)) + raise CommandExecutionError(f"Invalid Hive: {local_hive}") access_mask = registry.registry_32[use_32bit_registry] handle = None values = list() @@ -445,10 +444,10 @@ def list_values(hive, key=None, use_32bit_registry=False): except win32api.error as exc: if exc.winerror == 2: log.debug(r"Cannot find key: %s\%s", hive, key) - return False, r"Cannot find key: {}\{}".format(hive, key) + return False, rf"Cannot find key: {hive}\{key}" elif exc.winerror == 5: log.debug(r"Access is denied: %s\%s", hive, key) - return False, r"Access is denied: {}\{}".format(hive, key) + return False, rf"Access is denied: {hive}\{key}" raise finally: @@ -538,7 +537,7 @@ def read_value(hive, key, vname=None, use_32bit_registry=False): try: hkey = registry.hkeys[local_hive] except KeyError: - raise CommandExecutionError("Invalid Hive: {}".format(local_hive)) + raise CommandExecutionError(f"Invalid Hive: {local_hive}") access_mask = registry.registry_32[use_32bit_registry] handle = None @@ -574,13 +573,13 @@ def read_value(hive, key, vname=None, use_32bit_registry=False): raise except win32api.error as exc: if exc.winerror == 2: - msg = "Cannot find key: {}\\{}".format(local_hive, local_key) + msg = f"Cannot find key: {local_hive}\\{local_key}" log.trace(exc) log.trace(msg) ret["comment"] = msg ret["success"] = False elif exc.winerror == 5: - msg = "Access is denied: {}\\{}".format(local_hive, local_key) + msg = f"Access is denied: {local_hive}\\{local_key}" log.trace(exc) log.trace(msg) ret["comment"] = msg @@ -733,7 +732,7 @@ def set_value( try: hkey = registry.hkeys[local_hive] except KeyError: - raise CommandExecutionError("Invalid Hive: {}".format(local_hive)) + raise CommandExecutionError(f"Invalid Hive: {local_hive}") vtype_value = registry.vtype[local_vtype] access_mask = registry.registry_32[use_32bit_registry] | win32con.KEY_ALL_ACCESS @@ -886,7 +885,7 @@ def delete_key_recursive(hive, key, use_32bit_registry=False): try: hkey = registry.hkeys[local_hive] except KeyError: - raise CommandExecutionError("Invalid Hive: {}".format(local_hive)) + raise CommandExecutionError(f"Invalid Hive: {local_hive}") key_path = local_key access_mask = registry.registry_32[use_32bit_registry] | win32con.KEY_ALL_ACCESS @@ -918,7 +917,7 @@ def delete_key_recursive(hive, key, use_32bit_registry=False): """ _key = win32api.RegOpenKeyEx(_hkey, _keypath, 0, _access_mask) for subkeyname in _subkeys(_key): - subkeypath = "{}\\{}".format(_keypath, subkeyname) + subkeypath = f"{_keypath}\\{subkeyname}" _ret = _traverse_registry_tree(_hkey, subkeypath, _ret, access_mask) _ret.append(subkeypath) return _ret @@ -938,13 +937,13 @@ def delete_key_recursive(hive, key, use_32bit_registry=False): key_handle = win32api.RegOpenKeyEx(hkey, sub_key_path, 0, access_mask) try: win32api.RegDeleteKey(key_handle, "") - ret["Deleted"].append(r"{}\{}".format(hive, sub_key_path)) + ret["Deleted"].append(rf"{hive}\{sub_key_path}") except OSError as exc: log.error(exc, exc_info=True) - ret["Failed"].append(r"{}\{} {}".format(hive, sub_key_path, exc)) + ret["Failed"].append(rf"{hive}\{sub_key_path} {exc}") except win32api.error as exc: log.error(exc, exc_info=True) - ret["Failed"].append(r"{}\{} {}".format(hive, sub_key_path, exc.strerror)) + ret["Failed"].append(rf"{hive}\{sub_key_path} {exc.strerror}") finally: if key_handle: win32api.RegCloseKey(key_handle) @@ -998,7 +997,7 @@ def delete_value(hive, key, vname=None, use_32bit_registry=False): try: hkey = registry.hkeys[local_hive] except KeyError: - raise CommandExecutionError("Invalid Hive: {}".format(local_hive)) + raise CommandExecutionError(f"Invalid Hive: {local_hive}") access_mask = registry.registry_32[use_32bit_registry] | win32con.KEY_ALL_ACCESS handle = None diff --git a/tests/pytests/unit/test__init__.py b/tests/pytests/unit/test__init__.py new file mode 100644 index 00000000000..9c4df804c58 --- /dev/null +++ b/tests/pytests/unit/test__init__.py @@ -0,0 +1,6 @@ +import sys + + +def test_salt_system_encoding(): + encoding = sys.getdefaultencoding() + assert __salt_system_encoding__ == encoding From 69952a8ff45da1806af53dfc550277de732cfbcb Mon Sep 17 00:00:00 2001 From: Shane Lee Date: Wed, 10 Jan 2024 16:46:51 -0700 Subject: [PATCH 09/29] Add encoding option to the ini module Adds a new encoding parameter to the ini module to handle scenarios where the ini file encoding is not utf-8. --- salt/modules/ini_manage.py | 45 ++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/salt/modules/ini_manage.py b/salt/modules/ini_manage.py index 3f6042fcc2f..b541d1ac9f4 100644 --- a/salt/modules/ini_manage.py +++ b/salt/modules/ini_manage.py @@ -37,7 +37,7 @@ COM_REGX = re.compile(r"^\s*(#|;)\s*(.*)") INDENTED_REGX = re.compile(r"(\s+)(.*)") -def set_option(file_name, sections=None, separator="="): +def set_option(file_name, sections=None, separator="=", encoding=None): """ Edit an ini file, replacing one or more sections. Returns a dictionary containing the changes made. @@ -77,13 +77,13 @@ def set_option(file_name, sections=None, separator="="): """ sections = sections or {} changes = {} - inifile = _Ini.get_ini_file(file_name, separator=separator) + inifile = _Ini.get_ini_file(file_name, separator=separator, encoding=encoding) changes = inifile.update(sections) inifile.flush() return changes -def get_option(file_name, section, option, separator="="): +def get_option(file_name, section, option, separator="=", encoding=None): """ Get value of a key from a section in an ini file. Returns ``None`` if no matching key was found. @@ -103,7 +103,7 @@ def get_option(file_name, section, option, separator="="): salt '*' ini.get_option /path/to/ini section_name option_name """ - inifile = _Ini.get_ini_file(file_name, separator=separator) + inifile = _Ini.get_ini_file(file_name, separator=separator, encoding=encoding) if section: try: return inifile.get(section, {}).get(option, None) @@ -113,7 +113,7 @@ def get_option(file_name, section, option, separator="="): return inifile.get(option, None) -def remove_option(file_name, section, option, separator="="): +def remove_option(file_name, section, option, separator="=", encoding=None): """ Remove a key/value pair from a section in an ini file. Returns the value of the removed key, or ``None`` if nothing was removed. @@ -133,7 +133,7 @@ def remove_option(file_name, section, option, separator="="): salt '*' ini.remove_option /path/to/ini section_name option_name """ - inifile = _Ini.get_ini_file(file_name, separator=separator) + inifile = _Ini.get_ini_file(file_name, separator=separator, encoding=encoding) if isinstance(inifile.get(section), (dict, OrderedDict)): value = inifile.get(section, {}).pop(option, None) else: @@ -142,7 +142,7 @@ def remove_option(file_name, section, option, separator="="): return value -def get_section(file_name, section, separator="="): +def get_section(file_name, section, separator="=", encoding=None): """ Retrieve a section from an ini file. Returns the section as dictionary. If the section is not found, an empty dictionary is returned. @@ -162,7 +162,7 @@ def get_section(file_name, section, separator="="): salt '*' ini.get_section /path/to/ini section_name """ - inifile = _Ini.get_ini_file(file_name, separator=separator) + inifile = _Ini.get_ini_file(file_name, separator=separator, encoding=encoding) ret = {} for key, value in inifile.get(section, {}).items(): if key[0] != "#": @@ -170,7 +170,7 @@ def get_section(file_name, section, separator="="): return ret -def remove_section(file_name, section, separator="="): +def remove_section(file_name, section, separator="=", encoding=None): """ Remove a section in an ini file. Returns the removed section as dictionary, or ``None`` if nothing was removed. @@ -190,7 +190,7 @@ def remove_section(file_name, section, separator="="): salt '*' ini.remove_section /path/to/ini section_name """ - inifile = _Ini.get_ini_file(file_name, separator=separator) + inifile = _Ini.get_ini_file(file_name, separator=separator, encoding=encoding) if section in inifile: section = inifile.pop(section) inifile.flush() @@ -201,7 +201,7 @@ def remove_section(file_name, section, separator="="): return ret -def get_ini(file_name, separator="="): +def get_ini(file_name, separator="=", encoding=None): """ Retrieve whole structure from an ini file and return it as dictionary. @@ -236,7 +236,7 @@ def get_ini(file_name, separator="="): ret.update({key: val}) return ret - inifile = _Ini.get_ini_file(file_name, separator=separator) + inifile = _Ini.get_ini_file(file_name, separator=separator, encoding=encoding) return ini_odict2dict(inifile) @@ -409,13 +409,24 @@ class _Section(OrderedDict): class _Ini(_Section): + def __init__( + self, name, inicontents="", separator="=", commenter="#", encoding=None + ): + super().__init__( + self, inicontents=inicontents, separator=separator, commenter=commenter + ) + self.name = name + if encoding is None: + encoding = __salt_system_encoding__ + self.encoding = encoding + def refresh(self, inicontents=None): if inicontents is None: if not os.path.exists(self.name): log.trace("File %s does not exist and will be created", self.name) return try: - with salt.utils.files.fopen(self.name) as rfh: + with salt.utils.files.fopen(self.name, encoding=self.encoding) as rfh: inicontents = salt.utils.stringutils.to_unicode(rfh.read()) inicontents = os.linesep.join(inicontents.splitlines()) except OSError as exc: @@ -443,7 +454,9 @@ class _Ini(_Section): def flush(self): try: - with salt.utils.files.fopen(self.name, "wb") as outfile: + with salt.utils.files.fopen( + self.name, "wb", encoding=self.encoding + ) as outfile: ini_gen = self.gen_ini() next(ini_gen) ini_gen_list = list(ini_gen) @@ -457,8 +470,8 @@ class _Ini(_Section): ) @staticmethod - def get_ini_file(file_name, separator="="): - inifile = _Ini(file_name, separator=separator) + def get_ini_file(file_name, separator="=", encoding=None): + inifile = _Ini(file_name, separator=separator, encoding=encoding) inifile.refresh() return inifile From 51adf9e0b270d2fb8055abab3a75fcfd61c9a897 Mon Sep 17 00:00:00 2001 From: Shane Lee Date: Thu, 11 Jan 2024 17:10:34 -0700 Subject: [PATCH 10/29] Migrate ini_manage tests from unittests --- salt/modules/ini_manage.py | 8 +- .../integration/states/test_ini_manage.py | 4 + tests/pytests/unit/modules/test_ini_manage.py | 242 +++++++++++++++++- tests/unit/modules/test_ini_manage.py | 232 ----------------- 4 files changed, 248 insertions(+), 238 deletions(-) delete mode 100644 tests/unit/modules/test_ini_manage.py diff --git a/salt/modules/ini_manage.py b/salt/modules/ini_manage.py index b541d1ac9f4..e59708216eb 100644 --- a/salt/modules/ini_manage.py +++ b/salt/modules/ini_manage.py @@ -454,16 +454,16 @@ class _Ini(_Section): def flush(self): try: - with salt.utils.files.fopen( - self.name, "wb", encoding=self.encoding - ) as outfile: + with salt.utils.files.fopen(self.name, "wb") as outfile: ini_gen = self.gen_ini() next(ini_gen) ini_gen_list = list(ini_gen) # Avoid writing an initial line separator. if ini_gen_list: ini_gen_list[0] = ini_gen_list[0].lstrip(os.linesep) - outfile.writelines(salt.utils.data.encode(ini_gen_list)) + outfile.writelines( + salt.utils.data.encode(ini_gen_list, encoding=self.encoding) + ) except OSError as exc: raise CommandExecutionError( "Unable to write file '{}'. Exception: {}".format(self.name, exc) diff --git a/tests/pytests/integration/states/test_ini_manage.py b/tests/pytests/integration/states/test_ini_manage.py index 7558f2e6eb3..c67eb9df180 100644 --- a/tests/pytests/integration/states/test_ini_manage.py +++ b/tests/pytests/integration/states/test_ini_manage.py @@ -4,6 +4,10 @@ Integration tests for the ini_manage state import pytest +pytestmark = [ + pytest.mark.windows_whitelisted, +] + def test_options_present(salt_master, salt_call_cli): """ diff --git a/tests/pytests/unit/modules/test_ini_manage.py b/tests/pytests/unit/modules/test_ini_manage.py index b76dfa32140..a9ad0b7aa6b 100644 --- a/tests/pytests/unit/modules/test_ini_manage.py +++ b/tests/pytests/unit/modules/test_ini_manage.py @@ -1,6 +1,49 @@ import os -import salt.modules.ini_manage +import pytest + +import salt.modules.ini_manage as ini +import salt.utils.files +import salt.utils.stringutils + + +@pytest.fixture +def ini_content(): + return [ + "# Comment on the first line", + "", + "# First main option", + "option1=main1", + "", + "# Second main option", + "option2=main2", + "", + "", + "[main]", + "# Another comment", + "test1=value 1", + "", + "test2=value 2", + "", + "[SectionB]", + "test1=value 1B", + "", + "# Blank line should be above", + "test3 = value 3B", + "", + "[SectionC]", + "# The following option is empty", + "empty_option=", + ] + + +@pytest.fixture(scope="function") +def ini_file(tmp_path, ini_content): + file_path = tmp_path / "file.ini" + yield file_path + + +# def ini_file_linesep(ini_file, linesep): def test_section_req(): @@ -8,4 +51,199 @@ def test_section_req(): Test the __repr__ in the _Section class """ expected = "_Section(){}{{}}".format(os.linesep) - assert repr(salt.modules.ini_manage._Section("test")) == expected + assert repr(ini._Section("test")) == expected + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +def test_get_option(linesep, ini_file, ini_content): + """ + Test get_option method. + """ + content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + ini_file.write_bytes(content) + + assert ini.get_option(str(ini_file), "main", "test1") == "value 1" + assert ini.get_option(str(ini_file), "main", "test2") == "value 2" + assert ini.get_option(str(ini_file), "SectionB", "test1") == "value 1B" + assert ini.get_option(str(ini_file), "SectionB", "test3") == "value 3B" + assert ini.get_option(str(ini_file), "SectionC", "empty_option") == "" + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +def test_get_section(linesep, ini_file, ini_content): + """ + Test get_section method. + """ + content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + ini_file.write_bytes(content) + + expected = {"test1": "value 1B", "test3": "value 3B"} + assert ini.get_section(str(ini_file), "SectionB") == expected + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +def test_remove_option(linesep, ini_file, ini_content): + """ + Test remove_option method. + """ + content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + ini_file.write_bytes(content) + + assert ini.remove_option(str(ini_file), "SectionB", "test1") == "value 1B" + assert ini.get_option(str(ini_file), "SectionB", "test1") is None + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +def test_remove_section(linesep, ini_file, ini_content): + """ + Test remove_section method. + """ + content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + ini_file.write_bytes(content) + + expected = {"test1": "value 1B", "test3": "value 3B"} + assert ini.remove_section(str(ini_file), "SectionB") == expected + assert ini.get_section(str(ini_file), "SectionB") == {} + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +def test_get_ini(linesep, ini_file, ini_content): + """ + Test get_ini method. + """ + content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + ini_file.write_bytes(content) + + expected = { + "SectionC": { + "empty_option": "", + }, + "SectionB": { + "test1": "value 1B", + "test3": "value 3B", + }, + "main": { + "test1": "value 1", + "test2": "value 2", + }, + "option2": "main2", + "option1": "main1", + } + assert dict(ini.get_ini(str(ini_file))) == expected + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +def test_set_option(linesep, ini_file, ini_content): + """ + Test set_option method. + """ + content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + ini_file.write_bytes(content) + + result = ini.set_option( + str(ini_file), + { + "SectionB": { + "test3": "new value 3B", + "test_set_option": "test_set_value", + }, + "SectionD": {"test_set_option2": "test_set_value1"}, + }, + ) + expected = { + "SectionB": { + "test3": {"after": "new value 3B", "before": "value 3B"}, + "test_set_option": {"after": "test_set_value", "before": None}, + }, + "SectionD": { + "after": {"test_set_option2": "test_set_value1"}, + "before": None, + }, + } + assert result == expected + + # Check existing option updated + assert ini.get_option(str(ini_file), "SectionB", "test3") == "new value 3B" + + # Check new section and option added + assert ( + ini.get_option(str(ini_file), "SectionD", "test_set_option2") + == "test_set_value1" + ) + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +def test_empty_value(linesep, ini_file, ini_content): + """ + Test empty value preserved after edit + """ + content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + ini_file.write_bytes(content) + + ini.set_option(str(ini_file), {"SectionB": {"test3": "new value 3B"}}) + with salt.utils.files.fopen(str(ini_file), "r") as fp_: + file_content = salt.utils.stringutils.to_unicode(fp_.read()) + expected = "{0}{1}{0}".format(os.linesep, "empty_option = ") + assert expected in file_content, "empty_option was not preserved" + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +def test_empty_lines(linesep, ini_file, ini_content): + """ + Test empty lines preserved after edit + """ + content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + ini_file.write_bytes(content) + + expected = os.linesep.join( + [ + "# Comment on the first line", + "", + "# First main option", + "option1 = main1", + "", + "# Second main option", + "option2 = main2", + "", + "[main]", + "# Another comment", + "test1 = value 1", + "", + "test2 = value 2", + "", + "[SectionB]", + "test1 = value 1B", + "", + "# Blank line should be above", + "test3 = new value 3B", + "", + "[SectionC]", + "# The following option is empty", + "empty_option = ", + "", + ] + ) + ini.set_option(str(ini_file), {"SectionB": {"test3": "new value 3B"}}) + with salt.utils.files.fopen(str(ini_file), "r") as fp_: + file_content = fp_.read() + assert expected == file_content + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +def test_empty_lines_multiple_edits(linesep, ini_file, ini_content): + """ + Test empty lines preserved after multiple edits + """ + ini.set_option( + str(ini_file), + {"SectionB": {"test3": "this value will be edited two times"}}, + ) + test_empty_lines(linesep, ini_file, ini_content) + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +def test_different_encoding(linesep, ini_file, ini_content): + """ + Test ability to read a different encoding + """ + assert True diff --git a/tests/unit/modules/test_ini_manage.py b/tests/unit/modules/test_ini_manage.py deleted file mode 100644 index 6ce05cf766d..00000000000 --- a/tests/unit/modules/test_ini_manage.py +++ /dev/null @@ -1,232 +0,0 @@ -""" -Testing ini_manage exec module. -""" - -import os -import tempfile - -import salt.modules.ini_manage as ini -import salt.utils.files -import salt.utils.stringutils -from tests.support.unit import TestCase - - -class IniManageTestCase(TestCase): - """ - Testing ini_manage exec module. - """ - - TEST_FILE_CONTENT = [ - "# Comment on the first line", - "", - "# First main option", - "option1=main1", - "", - "# Second main option", - "option2=main2", - "", - "", - "[main]", - "# Another comment", - "test1=value 1", - "", - "test2=value 2", - "", - "[SectionB]", - "test1=value 1B", - "", - "# Blank line should be above", - "test3 = value 3B", - "", - "[SectionC]", - "# The following option is empty", - "empty_option=", - ] - - maxDiff = None - - def _setUp(self, linesep): - self.tfile = tempfile.NamedTemporaryFile(delete=False, mode="w+b") - self.tfile.write( - salt.utils.stringutils.to_bytes(linesep.join(self.TEST_FILE_CONTENT)) - ) - self.tfile.close() - - def setUp(self): - self._setUp(os.linesep) - - def tearDown(self): - os.remove(self.tfile.name) - - def test_get_option(self): - """ - Test get_option method. - """ - self.assertEqual(ini.get_option(self.tfile.name, "main", "test1"), "value 1") - self.assertEqual(ini.get_option(self.tfile.name, "main", "test2"), "value 2") - self.assertEqual( - ini.get_option(self.tfile.name, "SectionB", "test1"), "value 1B" - ) - self.assertEqual( - ini.get_option(self.tfile.name, "SectionB", "test3"), "value 3B" - ) - self.assertEqual( - ini.get_option(self.tfile.name, "SectionC", "empty_option"), "" - ) - - def test_get_section(self): - """ - Test get_section method. - """ - self.assertEqual( - ini.get_section(self.tfile.name, "SectionB"), - {"test1": "value 1B", "test3": "value 3B"}, - ) - - def test_remove_option(self): - """ - Test remove_option method. - """ - self.assertEqual( - ini.remove_option(self.tfile.name, "SectionB", "test1"), "value 1B" - ) - self.assertIsNone(ini.get_option(self.tfile.name, "SectionB", "test1")) - - def test_remove_section(self): - """ - Test remove_section method. - """ - self.assertEqual( - ini.remove_section(self.tfile.name, "SectionB"), - {"test1": "value 1B", "test3": "value 3B"}, - ) - self.assertEqual(ini.get_section(self.tfile.name, "SectionB"), {}) - - def test_get_ini(self): - """ - Test get_ini method. - """ - self.assertEqual( - dict(ini.get_ini(self.tfile.name)), - { - "SectionC": {"empty_option": ""}, - "SectionB": {"test1": "value 1B", "test3": "value 3B"}, - "main": {"test1": "value 1", "test2": "value 2"}, - "option2": "main2", - "option1": "main1", - }, - ) - - def test_set_option(self): - """ - Test set_option method. - """ - result = ini.set_option( - self.tfile.name, - { - "SectionB": { - "test3": "new value 3B", - "test_set_option": "test_set_value", - }, - "SectionD": {"test_set_option2": "test_set_value1"}, - }, - ) - self.assertEqual( - result, - { - "SectionB": { - "test3": {"after": "new value 3B", "before": "value 3B"}, - "test_set_option": {"after": "test_set_value", "before": None}, - }, - "SectionD": { - "after": {"test_set_option2": "test_set_value1"}, - "before": None, - }, - }, - ) - # Check existing option updated - self.assertEqual( - ini.get_option(self.tfile.name, "SectionB", "test3"), "new value 3B" - ) - # Check new section and option added - self.assertEqual( - ini.get_option(self.tfile.name, "SectionD", "test_set_option2"), - "test_set_value1", - ) - - def test_empty_value(self): - """ - Test empty value preserved after edit - """ - ini.set_option(self.tfile.name, {"SectionB": {"test3": "new value 3B"}}) - with salt.utils.files.fopen(self.tfile.name, "r") as fp_: - file_content = salt.utils.stringutils.to_unicode(fp_.read()) - expected = "{0}{1}{0}".format(os.linesep, "empty_option = ") - self.assertIn(expected, file_content, "empty_option was not preserved") - - def test_empty_lines(self): - """ - Test empty lines preserved after edit - """ - ini.set_option(self.tfile.name, {"SectionB": {"test3": "new value 3B"}}) - expected = os.linesep.join( - [ - "# Comment on the first line", - "", - "# First main option", - "option1 = main1", - "", - "# Second main option", - "option2 = main2", - "", - "[main]", - "# Another comment", - "test1 = value 1", - "", - "test2 = value 2", - "", - "[SectionB]", - "test1 = value 1B", - "", - "# Blank line should be above", - "test3 = new value 3B", - "", - "[SectionC]", - "# The following option is empty", - "empty_option = ", - "", - ] - ) - with salt.utils.files.fopen(self.tfile.name, "r") as fp_: - file_content = salt.utils.stringutils.to_unicode(fp_.read()) - self.assertEqual(expected, file_content) - - def test_empty_lines_multiple_edits(self): - """ - Test empty lines preserved after multiple edits - """ - ini.set_option( - self.tfile.name, - {"SectionB": {"test3": "this value will be edited two times"}}, - ) - self.test_empty_lines() - - def test_newline_characters(self): - """ - Test newline characters - """ - for c in ["\n", "\r", "\r\n"]: - for test in [ - self.test_get_option, - self.test_get_section, - self.test_remove_option, - self.test_remove_section, - self.test_get_ini, - self.test_set_option, - self.test_empty_value, - self.test_empty_lines, - self.test_empty_lines_multiple_edits, - ]: - self.tearDown() - self._setUp(c) - test() From 4b6aad6230616bb4782483d2717adbf02b9948e6 Mon Sep 17 00:00:00 2001 From: Shane Lee Date: Fri, 12 Jan 2024 14:37:57 -0700 Subject: [PATCH 11/29] Add some tests for unicode Test reading and writing to utf-8, utf-16, utf-16-le and utf-32-le ini files --- salt/modules/ini_manage.py | 15 +- salt/utils/data.py | 2 +- tests/pytests/unit/modules/test_ini_manage.py | 335 ++++++++++++++++-- 3 files changed, 306 insertions(+), 46 deletions(-) diff --git a/salt/modules/ini_manage.py b/salt/modules/ini_manage.py index e59708216eb..bd90061b89b 100644 --- a/salt/modules/ini_manage.py +++ b/salt/modules/ini_manage.py @@ -76,7 +76,6 @@ def set_option(file_name, sections=None, separator="=", encoding=None): salt '*' ini.set_option /path/to/ini '{section_foo: {key: value}}' """ sections = sections or {} - changes = {} inifile = _Ini.get_ini_file(file_name, separator=separator, encoding=encoding) changes = inifile.update(sections) inifile.flush() @@ -426,8 +425,10 @@ class _Ini(_Section): log.trace("File %s does not exist and will be created", self.name) return try: - with salt.utils.files.fopen(self.name, encoding=self.encoding) as rfh: - inicontents = salt.utils.stringutils.to_unicode(rfh.read()) + with salt.utils.files.fopen( + self.name, "r", encoding=self.encoding + ) as rfh: + inicontents = rfh.read() inicontents = os.linesep.join(inicontents.splitlines()) except OSError as exc: if __opts__["test"] is False: @@ -454,16 +455,16 @@ class _Ini(_Section): def flush(self): try: - with salt.utils.files.fopen(self.name, "wb") as outfile: + with salt.utils.files.fopen( + self.name, "w", encoding=self.encoding + ) as outfile: ini_gen = self.gen_ini() next(ini_gen) ini_gen_list = list(ini_gen) # Avoid writing an initial line separator. if ini_gen_list: ini_gen_list[0] = ini_gen_list[0].lstrip(os.linesep) - outfile.writelines( - salt.utils.data.encode(ini_gen_list, encoding=self.encoding) - ) + outfile.writelines(ini_gen_list) except OSError as exc: raise CommandExecutionError( "Unable to write file '{}'. Exception: {}".format(self.name, exc) diff --git a/salt/utils/data.py b/salt/utils/data.py index f50ebbab789..0235551462f 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -169,7 +169,7 @@ def _remove_circular_refs(ob, _seen=None): This has been taken from author Martijn Pieters https://stackoverflow.com/questions/44777369/ remove-circular-references-in-dicts-lists-tuples/44777477#44777477 - :param ob: dict, list, typle, set, and frozenset + :param ob: dict, list, tuple, set, and frozenset Standard python object :param object _seen: Object that has circular reference diff --git a/tests/pytests/unit/modules/test_ini_manage.py b/tests/pytests/unit/modules/test_ini_manage.py index a9ad0b7aa6b..538e75c171b 100644 --- a/tests/pytests/unit/modules/test_ini_manage.py +++ b/tests/pytests/unit/modules/test_ini_manage.py @@ -1,4 +1,5 @@ import os +import sys import pytest @@ -43,7 +44,33 @@ def ini_file(tmp_path, ini_content): yield file_path -# def ini_file_linesep(ini_file, linesep): +@pytest.fixture +def unicode_content(): + return [ + "# An ini file with some unicode characters", + "", + "[Ascii]", + "de = Deutsch", + "en_GB = English (UK)", + "en_US = English (US)", + "fi = Suomi", + "hu = Magyar", + "it = Italiano", + "nl = Dutch", + "pt = Portuguese", + "sv = Svenska", + "", + "[Юникод]", + "# This means unicode in Russian", + "es = Español", + "es_ES = Español (ES)", + "fr = Français", + "hi = हिंदी", + "ja = 日本語", + "ko = :한국어", + "zh = 简体中文", + "繁體中文 = zh_TW", + ] def test_section_req(): @@ -55,63 +82,104 @@ def test_section_req(): @pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) -def test_get_option(linesep, ini_file, ini_content): +@pytest.mark.parametrize( + "encoding", [None, "cp1252" if sys.platform == "win32" else "ISO-2022-JP"] +) +def test_get_option(encoding, linesep, ini_file, ini_content): """ Test get_option method. """ - content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + content = salt.utils.stringutils.to_bytes( + linesep.join(ini_content), encoding=encoding + ) ini_file.write_bytes(content) - assert ini.get_option(str(ini_file), "main", "test1") == "value 1" - assert ini.get_option(str(ini_file), "main", "test2") == "value 2" - assert ini.get_option(str(ini_file), "SectionB", "test1") == "value 1B" - assert ini.get_option(str(ini_file), "SectionB", "test3") == "value 3B" - assert ini.get_option(str(ini_file), "SectionC", "empty_option") == "" + assert ( + ini.get_option(str(ini_file), "main", "test1", encoding=encoding) == "value 1" + ) + assert ( + ini.get_option(str(ini_file), "main", "test2", encoding=encoding) == "value 2" + ) + assert ( + ini.get_option(str(ini_file), "SectionB", "test1", encoding=encoding) + == "value 1B" + ) + assert ( + ini.get_option(str(ini_file), "SectionB", "test3", encoding=encoding) + == "value 3B" + ) + assert ( + ini.get_option(str(ini_file), "SectionC", "empty_option", encoding=encoding) + == "" + ) @pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) -def test_get_section(linesep, ini_file, ini_content): +@pytest.mark.parametrize( + "encoding", [None, "cp1252" if sys.platform == "win32" else "ISO-2022-JP"] +) +def test_get_section(encoding, linesep, ini_file, ini_content): """ Test get_section method. """ - content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + content = salt.utils.stringutils.to_bytes( + linesep.join(ini_content), encoding=encoding + ) ini_file.write_bytes(content) expected = {"test1": "value 1B", "test3": "value 3B"} - assert ini.get_section(str(ini_file), "SectionB") == expected + assert ini.get_section(str(ini_file), "SectionB", encoding=encoding) == expected @pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) -def test_remove_option(linesep, ini_file, ini_content): +@pytest.mark.parametrize( + "encoding", [None, "cp1252" if sys.platform == "win32" else "ISO-2022-JP"] +) +def test_remove_option(encoding, linesep, ini_file, ini_content): """ Test remove_option method. """ - content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + content = salt.utils.stringutils.to_bytes( + linesep.join(ini_content), encoding=encoding + ) ini_file.write_bytes(content) - assert ini.remove_option(str(ini_file), "SectionB", "test1") == "value 1B" - assert ini.get_option(str(ini_file), "SectionB", "test1") is None + assert ( + ini.remove_option(str(ini_file), "SectionB", "test1", encoding=encoding) + == "value 1B" + ) + assert ini.get_option(str(ini_file), "SectionB", "test1", encoding=encoding) is None @pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) -def test_remove_section(linesep, ini_file, ini_content): +@pytest.mark.parametrize( + "encoding", [None, "cp1252" if sys.platform == "win32" else "ISO-2022-JP"] +) +def test_remove_section(encoding, linesep, ini_file, ini_content): """ Test remove_section method. """ - content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + content = salt.utils.stringutils.to_bytes( + linesep.join(ini_content), encoding=encoding + ) ini_file.write_bytes(content) expected = {"test1": "value 1B", "test3": "value 3B"} - assert ini.remove_section(str(ini_file), "SectionB") == expected - assert ini.get_section(str(ini_file), "SectionB") == {} + assert ini.remove_section(str(ini_file), "SectionB", encoding=encoding) == expected + assert ini.get_section(str(ini_file), "SectionB", encoding=encoding) == {} @pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) -def test_get_ini(linesep, ini_file, ini_content): +@pytest.mark.parametrize( + "encoding", [None, "cp1252" if sys.platform == "win32" else "ISO-2022-JP"] +) +def test_get_ini(encoding, linesep, ini_file, ini_content): """ Test get_ini method. """ - content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + content = salt.utils.stringutils.to_bytes( + linesep.join(ini_content), encoding=encoding + ) ini_file.write_bytes(content) expected = { @@ -129,15 +197,20 @@ def test_get_ini(linesep, ini_file, ini_content): "option2": "main2", "option1": "main1", } - assert dict(ini.get_ini(str(ini_file))) == expected + assert dict(ini.get_ini(str(ini_file), encoding=encoding)) == expected @pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) -def test_set_option(linesep, ini_file, ini_content): +@pytest.mark.parametrize( + "encoding", [None, "cp1252" if sys.platform == "win32" else "ISO-2022-JP"] +) +def test_set_option(encoding, linesep, ini_file, ini_content): """ Test set_option method. """ - content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + content = salt.utils.stringutils.to_bytes( + linesep.join(ini_content), encoding=encoding + ) ini_file.write_bytes(content) result = ini.set_option( @@ -149,6 +222,7 @@ def test_set_option(linesep, ini_file, ini_content): }, "SectionD": {"test_set_option2": "test_set_value1"}, }, + encoding=encoding, ) expected = { "SectionB": { @@ -163,36 +237,51 @@ def test_set_option(linesep, ini_file, ini_content): assert result == expected # Check existing option updated - assert ini.get_option(str(ini_file), "SectionB", "test3") == "new value 3B" + assert ( + ini.get_option(str(ini_file), "SectionB", "test3", encoding=encoding) + == "new value 3B" + ) # Check new section and option added assert ( - ini.get_option(str(ini_file), "SectionD", "test_set_option2") + ini.get_option(str(ini_file), "SectionD", "test_set_option2", encoding=encoding) == "test_set_value1" ) @pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) -def test_empty_value(linesep, ini_file, ini_content): +@pytest.mark.parametrize( + "encoding", [None, "cp1252" if sys.platform == "win32" else "ISO-2022-JP"] +) +def test_empty_value(encoding, linesep, ini_file, ini_content): """ Test empty value preserved after edit """ - content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + content = salt.utils.stringutils.to_bytes( + linesep.join(ini_content), encoding=encoding + ) ini_file.write_bytes(content) - ini.set_option(str(ini_file), {"SectionB": {"test3": "new value 3B"}}) + ini.set_option( + str(ini_file), {"SectionB": {"test3": "new value 3B"}}, encoding=encoding + ) with salt.utils.files.fopen(str(ini_file), "r") as fp_: - file_content = salt.utils.stringutils.to_unicode(fp_.read()) + file_content = salt.utils.stringutils.to_unicode(fp_.read(), encoding=encoding) expected = "{0}{1}{0}".format(os.linesep, "empty_option = ") assert expected in file_content, "empty_option was not preserved" @pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) -def test_empty_lines(linesep, ini_file, ini_content): +@pytest.mark.parametrize( + "encoding", [None, "cp1252" if sys.platform == "win32" else "ISO-2022-JP"] +) +def test_empty_lines(encoding, linesep, ini_file, ini_content): """ Test empty lines preserved after edit """ - content = salt.utils.stringutils.to_bytes(linesep.join(ini_content)) + content = salt.utils.stringutils.to_bytes( + linesep.join(ini_content), encoding=encoding + ) ini_file.write_bytes(content) expected = os.linesep.join( @@ -223,27 +312,197 @@ def test_empty_lines(linesep, ini_file, ini_content): "", ] ) - ini.set_option(str(ini_file), {"SectionB": {"test3": "new value 3B"}}) + ini.set_option( + str(ini_file), {"SectionB": {"test3": "new value 3B"}}, encoding=encoding + ) with salt.utils.files.fopen(str(ini_file), "r") as fp_: file_content = fp_.read() assert expected == file_content @pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) -def test_empty_lines_multiple_edits(linesep, ini_file, ini_content): +@pytest.mark.parametrize( + "encoding", [None, "cp1252" if sys.platform == "win32" else "ISO-2022-JP"] +) +def test_empty_lines_multiple_edits(encoding, linesep, ini_file, ini_content): """ Test empty lines preserved after multiple edits """ + content = salt.utils.stringutils.to_bytes( + linesep.join(ini_content), encoding=encoding + ) + ini_file.write_bytes(content) + ini.set_option( str(ini_file), {"SectionB": {"test3": "this value will be edited two times"}}, + encoding=encoding, ) - test_empty_lines(linesep, ini_file, ini_content) + + expected = os.linesep.join( + [ + "# Comment on the first line", + "", + "# First main option", + "option1 = main1", + "", + "# Second main option", + "option2 = main2", + "", + "[main]", + "# Another comment", + "test1 = value 1", + "", + "test2 = value 2", + "", + "[SectionB]", + "test1 = value 1B", + "", + "# Blank line should be above", + "test3 = new value 3B", + "", + "[SectionC]", + "# The following option is empty", + "empty_option = ", + "", + ] + ) + ini.set_option( + str(ini_file), {"SectionB": {"test3": "new value 3B"}}, encoding=encoding + ) + with salt.utils.files.fopen(str(ini_file), "r") as fp_: + file_content = fp_.read() + assert expected == file_content @pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) -def test_different_encoding(linesep, ini_file, ini_content): +@pytest.mark.parametrize("encoding", [None, "utf-16", "utf-32-le"]) +def test_unicode_get_option(encoding, linesep, ini_file, unicode_content): """ - Test ability to read a different encoding + Test ability to get an option from a file that contains unicode characters + We can't encode the file with something that doesn't support unicode + Ie: cp1252 """ - assert True + content = salt.utils.stringutils.to_bytes( + linesep.join(unicode_content), encoding=encoding + ) + ini_file.write_bytes(content) + + # Get a non-unicode value + assert ini.get_option(str(ini_file), "Ascii", "de", encoding=encoding) == "Deutsch" + + # Get a unicode value + assert ini.get_option(str(ini_file), "Юникод", "hi", encoding=encoding) == "हिंदी" + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +@pytest.mark.parametrize("encoding", [None, "utf-16", "utf-16-le", "utf-32-le"]) +def test_unicode_set_option(encoding, linesep, ini_file, unicode_content): + """ + Test ability to set an option in a file that contains unicode characters. + The option itself may be unicode + We can't encode the file with something that doesn't support unicode + Ie: cp1252 + """ + content = salt.utils.stringutils.to_bytes( + linesep.join(unicode_content), encoding=encoding + ) + ini_file.write_bytes(content) + + result = ini.set_option( + str(ini_file), + { + "Ascii": {"ay": "Aymar"}, + "Юникод": {"dv": "ދިވެހިބަސް"}, + }, + encoding=encoding, + ) + expected = { + "Ascii": { + "ay": { + "before": None, + "after": "Aymar", + }, + }, + "Юникод": { + "dv": { + "before": None, + "after": "ދިވެހިބަސް", + }, + }, + } + assert result == expected + + # Check existing option updated + assert ini.get_option(str(ini_file), "Ascii", "ay", encoding=encoding) == "Aymar" + + # Check new section and option added + assert ( + ini.get_option(str(ini_file), "Юникод", "dv", encoding=encoding) == "ދިވެހިބަސް" + ) + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +@pytest.mark.parametrize("encoding", [None, "utf-16", "utf-16-le", "utf-32-le"]) +def test_unicode_get_section(encoding, linesep, ini_file, unicode_content): + """ + Test get_section method. + """ + content = salt.utils.stringutils.to_bytes( + linesep.join(unicode_content), encoding=encoding + ) + ini_file.write_bytes(content) + + expected = { + "es": "Español", + "es_ES": "Español (ES)", + "fr": "Français", + "hi": "हिंदी", + "ja": "日本語", + "ko": ":한국어", + "zh": "简体中文", + "繁體中文": "zh_TW", + } + assert ini.get_section(str(ini_file), "Юникод", encoding=encoding) == expected + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +@pytest.mark.parametrize("encoding", [None, "utf-16", "utf-16-le", "utf-32-le"]) +def test_unicode_remove_option(encoding, linesep, ini_file, unicode_content): + """ + Test remove_option method. + """ + content = salt.utils.stringutils.to_bytes( + linesep.join(unicode_content), encoding=encoding + ) + ini_file.write_bytes(content) + + assert ( + ini.remove_option(str(ini_file), "Юникод", "繁體中文", encoding=encoding) == "zh_TW" + ) + assert ini.get_option(str(ini_file), "Юникод", "繁體中文", encoding=encoding) is None + + +@pytest.mark.parametrize("linesep", ["\r", "\n", "\r\n"]) +@pytest.mark.parametrize("encoding", [None, "utf-16", "utf-16-le", "utf-32-le"]) +def test_unicode_remove_section(encoding, linesep, ini_file, unicode_content): + """ + Test remove_section method. + """ + content = salt.utils.stringutils.to_bytes( + linesep.join(unicode_content), encoding=encoding + ) + ini_file.write_bytes(content) + + expected = { + "es": "Español", + "es_ES": "Español (ES)", + "fr": "Français", + "hi": "हिंदी", + "ja": "日本語", + "ko": ":한국어", + "zh": "简体中文", + "繁體中文": "zh_TW", + } + assert ini.remove_section(str(ini_file), "Юникод", encoding=encoding) == expected + assert ini.get_section(str(ini_file), "Юникод", encoding=encoding) == {} From 166560c4c98125980ff12d554f71d1b6ff460dce Mon Sep 17 00:00:00 2001 From: Shane Lee Date: Fri, 12 Jan 2024 15:03:44 -0700 Subject: [PATCH 12/29] Add documentation to the ini_manage module Add missing documentation for all functions Add documentation for new parameter `encoding` Add Returns section to documentation --- salt/modules/ini_manage.py | 211 ++++++++++++++++++++++++++++++++----- 1 file changed, 182 insertions(+), 29 deletions(-) diff --git a/salt/modules/ini_manage.py b/salt/modules/ini_manage.py index bd90061b89b..70ce447f5b0 100644 --- a/salt/modules/ini_manage.py +++ b/salt/modules/ini_manage.py @@ -42,21 +42,32 @@ def set_option(file_name, sections=None, separator="=", encoding=None): Edit an ini file, replacing one or more sections. Returns a dictionary containing the changes made. - file_name - path of ini_file + Args: - sections : None - A dictionary representing the sections to be edited ini file - The keys are the section names and the values are the dictionary - containing the options - If the ini file does not contain sections the keys and values represent - the options + file_name (str): + The full path to the ini file. - separator : = - A character used to separate keys and values. Standard ini files use - the "=" character. + sections (dict): + A dictionary representing the sections to be edited in the ini file. + The keys are the section names and the values are a dictionary + containing the options. If the ini file does not contain sections + the keys and values represent the options. The default is ``None``. - .. versionadded:: 2016.11.0 + separator (str): + The character used to separate keys and values. Standard ini files + use the "=" character. The default is ``=``. + + .. versionadded:: 2016.11.0 + + encoding (str): + A string value representing encoding of the target ini file. If + ``None`` is passed, it uses the system default which is likely + ``utf-8``. Default is ``None`` + + .. versionadded:: 3006.6 + + Returns: + dict: A dictionary representing the changes made to the ini file API Example: @@ -65,8 +76,7 @@ def set_option(file_name, sections=None, separator="=", encoding=None): import salt.client with salt.client.get_local_client() as sc: sc.cmd( - 'target', 'ini.set_option', - ['path_to_ini_file', '{"section_to_change": {"key": "value"}}'] + 'target', 'ini.set_option', ['path_to_ini_file', '{"section_to_change": {"key": "value"}}'] ) CLI Example: @@ -74,7 +84,9 @@ def set_option(file_name, sections=None, separator="=", encoding=None): .. code-block:: bash salt '*' ini.set_option /path/to/ini '{section_foo: {key: value}}' + """ + sections = sections or {} inifile = _Ini.get_ini_file(file_name, separator=separator, encoding=encoding) changes = inifile.update(sections) @@ -87,21 +99,51 @@ def get_option(file_name, section, option, separator="=", encoding=None): Get value of a key from a section in an ini file. Returns ``None`` if no matching key was found. + Args: + + file_name (str): + The full path to the ini file. + + section (str): + A string value representing the section of the ini that the option + is in. If the option is not in a section, leave this empty. + + option (str): + A string value representing the option to search for. + + separator (str): + The character used to separate keys and values. Standard ini files + use the "=" character. The default is ``=``. + + .. versionadded:: 2016.11.0 + + encoding (str): + A string value representing encoding of the target ini file. If + ``None`` is passed, it uses the system default which is likely + ``utf-8``. Default is ``None`` + + .. versionadded:: 3006.6 + + Returns: + str: The value as defined in the ini file, or ``None`` if empty or not + found + API Example: .. code-block:: python import salt.client with salt.client.get_local_client() as sc: - sc.cmd('target', 'ini.get_option', - [path_to_ini_file, section_name, option]) + sc.cmd('target', 'ini.get_option', [path_to_ini_file, section_name, option]) CLI Example: .. code-block:: bash salt '*' ini.get_option /path/to/ini section_name option_name + """ + inifile = _Ini.get_ini_file(file_name, separator=separator, encoding=encoding) if section: try: @@ -117,21 +159,51 @@ def remove_option(file_name, section, option, separator="=", encoding=None): Remove a key/value pair from a section in an ini file. Returns the value of the removed key, or ``None`` if nothing was removed. + Args: + + file_name (str): + The full path to the ini file. + + section (str): + A string value representing the section of the ini that the option + is in. If the option is not in a section, leave this empty. + + option (str): + A string value representing the option to search for. + + separator (str): + The character used to separate keys and values. Standard ini files + use the "=" character. The default is ``=``. + + .. versionadded:: 2016.11.0 + + encoding (str): + A string value representing encoding of the target ini file. If + ``None`` is passed, it uses the system default which is likely + ``utf-8``. Default is ``None`` + + .. versionadded:: 3006.6 + + Returns: + str: A string value representing the option that was removed or ``None`` + if nothing was removed + API Example: .. code-block:: python import salt sc = salt.client.get_local_client() - sc.cmd('target', 'ini.remove_option', - [path_to_ini_file, section_name, option]) + sc.cmd('target', 'ini.remove_option', [path_to_ini_file, section_name, option]) CLI Example: .. code-block:: bash salt '*' ini.remove_option /path/to/ini section_name option_name + """ + inifile = _Ini.get_ini_file(file_name, separator=separator, encoding=encoding) if isinstance(inifile.get(section), (dict, OrderedDict)): value = inifile.get(section, {}).pop(option, None) @@ -143,24 +215,51 @@ def remove_option(file_name, section, option, separator="=", encoding=None): def get_section(file_name, section, separator="=", encoding=None): """ - Retrieve a section from an ini file. Returns the section as dictionary. If + Retrieve a section from an ini file. Returns the section as a dictionary. If the section is not found, an empty dictionary is returned. + Args: + + file_name (str): + The full path to the ini file. + + section (str): + A string value representing name of the section to search for. + + separator (str): + The character used to separate keys and values. Standard ini files + use the "=" character. The default is ``=``. + + .. versionadded:: 2016.11.0 + + encoding (str): + A string value representing encoding of the target ini file. If + ``None`` is passed, it uses the system default which is likely + ``utf-8``. Default is ``None`` + + .. versionadded:: 3006.6 + + Returns: + dict: A dictionary containing the names and values of all items in the + section of the ini file. If the section is not found, an empty + dictionary is returned + API Example: .. code-block:: python import salt.client with salt.client.get_local_client() as sc: - sc.cmd('target', 'ini.get_section', - [path_to_ini_file, section_name]) + sc.cmd('target', 'ini.get_section', [path_to_ini_file, section_name]) CLI Example: .. code-block:: bash salt '*' ini.get_section /path/to/ini section_name + """ + inifile = _Ini.get_ini_file(file_name, separator=separator, encoding=encoding) ret = {} for key, value in inifile.get(section, {}).items(): @@ -171,8 +270,33 @@ def get_section(file_name, section, separator="=", encoding=None): def remove_section(file_name, section, separator="=", encoding=None): """ - Remove a section in an ini file. Returns the removed section as dictionary, - or ``None`` if nothing was removed. + Remove a section in an ini file. Returns the removed section as a + dictionary, or ``None`` if nothing is removed. + + Args: + + file_name (str): + The full path to the ini file. + + section (str): + A string value representing the name of the section search for. + + separator (str): + The character used to separate keys and values. Standard ini files + use the "=" character. The default is ``=``. + + .. versionadded:: 2016.11.0 + + encoding (str): + A string value representing encoding of the target ini file. If + ``None`` is passed, it uses the system default which is likely + ``utf-8``. Default is ``None`` + + .. versionadded:: 3006.6 + + Returns: + dict: A dictionary containing the names and values of all items in the + section that was removed or ``None`` if nothing was removed API Example: @@ -180,15 +304,16 @@ def remove_section(file_name, section, separator="=", encoding=None): import salt.client with salt.client.get_local_client() as sc: - sc.cmd('target', 'ini.remove_section', - [path_to_ini_file, section_name]) + sc.cmd('target', 'ini.remove_section', [path_to_ini_file, section_name]) CLI Example: .. code-block:: bash salt '*' ini.remove_section /path/to/ini section_name + """ + inifile = _Ini.get_ini_file(file_name, separator=separator, encoding=encoding) if section in inifile: section = inifile.pop(section) @@ -202,22 +327,44 @@ def remove_section(file_name, section, separator="=", encoding=None): def get_ini(file_name, separator="=", encoding=None): """ - Retrieve whole structure from an ini file and return it as dictionary. + Retrieve the whole structure from an ini file and return it as a dictionary. + + Args: + + file_name (str): + The full path to the ini file. + + separator (str): + The character used to separate keys and values. Standard ini files + use the "=" character. The default is ``=``. + + .. versionadded:: 2016.11.0 + + encoding (str): + A string value representing encoding of the target ini file. If + ``None`` is passed, it uses the system default which is likely + ``utf-8``. Default is ``None`` + + .. versionadded:: 3006.6 + + Returns: + dict: A dictionary containing the sections along with the values and + names contained in each section API Example: .. code-block:: python import salt.client - with salt.client.giet_local_client() as sc: - sc.cmd('target', 'ini.get_ini', - [path_to_ini_file]) + with salt.client.get_local_client() as sc: + sc.cmd('target', 'ini.get_ini', [path_to_ini_file]) CLI Example: .. code-block:: bash salt '*' ini.get_ini /path/to/ini + """ def ini_odict2dict(odict): @@ -226,6 +373,7 @@ def get_ini(file_name, separator="=", encoding=None): :param odict: OrderedDict :return: regular dict """ + ret = {} for key, val in odict.items(): if key[0] != "#": @@ -425,6 +573,8 @@ class _Ini(_Section): log.trace("File %s does not exist and will be created", self.name) return try: + # We need to set decode on open and not try to do it later with + # stringutils with salt.utils.files.fopen( self.name, "r", encoding=self.encoding ) as rfh: @@ -455,6 +605,9 @@ class _Ini(_Section): def flush(self): try: + # We need to encode in the fopen command instead of using + # data.encode in the writelines command. Using data.encode will + # cause a BoM to be placed on every line of the file with salt.utils.files.fopen( self.name, "w", encoding=self.encoding ) as outfile: From bbff5cec155320733ffa3af9771f023a3279f26a Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 23 Jan 2024 18:44:04 +0000 Subject: [PATCH 13/29] Stop time bombing with `RuntimeError`'s Signed-off-by: Pedro Algarvio --- changelog/665924.changed.md | 1 + salt/utils/versions.py | 23 ++++++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 changelog/665924.changed.md diff --git a/changelog/665924.changed.md b/changelog/665924.changed.md new file mode 100644 index 00000000000..fb06f9125c4 --- /dev/null +++ b/changelog/665924.changed.md @@ -0,0 +1 @@ +Salt no longer time bombs user installations on code using `salt.utils.versions.warn_until_date` diff --git a/salt/utils/versions.py b/salt/utils/versions.py index 2f11766b223..9a97fee1bd7 100644 --- a/salt/utils/versions.py +++ b/salt/utils/versions.py @@ -6,12 +6,11 @@ which works under python 3 because on python 3 you can no longer compare strings against integers. """ - - import datetime import inspect import logging import numbers +import os import sys import warnings @@ -171,7 +170,7 @@ def warn_until( if _version_ >= version: caller = inspect.getframeinfo(sys._getframe(stacklevel - 1)) - raise RuntimeError( + deprecated_message = ( "The warning triggered on filename '{filename}', line number " "{lineno}, is supposed to be shown until version " "{until_version} is released. Current version is now " @@ -180,8 +179,15 @@ def warn_until( lineno=caller.lineno, until_version=version.formatted_version, salt_version=_version_.formatted_version, - ), + ) ) + if os.environ.get("RAISE_DEPRECATIONS_RUNTIME_ERRORS", "0") == "1": + # We don't raise RuntimeError by default since that can break + # users systems. We do however want to raise them in a CI context. + raise RuntimeError(deprecated_message) + # Otherwise, print the deprecated message to STDERR + sys.stderr.write(f"\n{deprecated_message}\n") + sys.stderr.flush() if _dont_call_warnings is False: warnings.warn( @@ -239,7 +245,7 @@ def warn_until_date( today = _current_date or datetime.datetime.utcnow().date() if today >= date: caller = inspect.getframeinfo(sys._getframe(stacklevel - 1)) - raise RuntimeError( + deprecated_message = ( "{message} This warning(now exception) triggered on " "filename '{filename}', line number {lineno}, is " "supposed to be shown until {date}. Today is {today}. " @@ -251,6 +257,13 @@ def warn_until_date( today=today.isoformat(), ), ) + if os.environ.get("RAISE_DEPRECATIONS_RUNTIME_ERRORS", "0") == "1": + # We don't raise RuntimeError by default since that can break + # users systems. We do however want to raise them in a CI context. + raise RuntimeError(deprecated_message) + # Otherwise, print the deprecated message to STDERR + sys.stderr.write(f"\n{deprecated_message}\n") + sys.stderr.flush() if _dont_call_warnings is False: warnings.warn( From 487a1ad3d09c8c7efed52df3f1311d4c1c001672 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 23 Jan 2024 18:53:17 +0000 Subject: [PATCH 14/29] Be sure to raise the `RuntimeError`'s in the CI context Signed-off-by: Pedro Algarvio --- .github/workflows/build-deps-ci-action.yml | 1 + .github/workflows/ci.yml | 1 + .github/workflows/nightly.yml | 1 + .github/workflows/release.yml | 1 + .github/workflows/scheduled.yml | 1 + .github/workflows/staging.yml | 1 + .github/workflows/templates/build-deps-ci-action.yml.jinja | 1 + .github/workflows/templates/layout.yml.jinja | 1 + .../workflows/templates/test-package-downloads-action.yml.jinja | 1 + .github/workflows/test-action-linux.yml | 1 + .github/workflows/test-action-macos.yml | 1 + .github/workflows/test-action-windows.yml | 1 + .github/workflows/test-package-downloads-action.yml | 1 + .github/workflows/test-packages-action-linux.yml | 1 + .github/workflows/test-packages-action-macos.yml | 1 + .github/workflows/test-packages-action-windows.yml | 1 + tools/vm.py | 1 + 17 files changed, 17 insertions(+) diff --git a/.github/workflows/build-deps-ci-action.yml b/.github/workflows/build-deps-ci-action.yml index 65f0263743f..82389dbb448 100644 --- a/.github/workflows/build-deps-ci-action.yml +++ b/.github/workflows/build-deps-ci-action.yml @@ -43,6 +43,7 @@ env: PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ PIP_EXTRA_INDEX_URL: https://pypi.org/simple PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" jobs: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1dd2f7d7c3..16573bf856b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,7 @@ env: CACHE_SEED: SEED-7 # Bump the number to invalidate all caches RELENV_DATA: "${{ github.workspace }}/.relenv" PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" permissions: contents: read # for dorny/paths-filter to fetch a list of changed files diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 276ff27c832..ae18627bc9d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -25,6 +25,7 @@ env: CACHE_SEED: SEED-7 # Bump the number to invalidate all caches RELENV_DATA: "${{ github.workspace }}/.relenv" PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" permissions: contents: read # for dorny/paths-filter to fetch a list of changed files diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 661a84340f5..87362595f48 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,7 @@ env: CACHE_SEED: SEED-7 # Bump the number to invalidate all caches RELENV_DATA: "${{ github.workspace }}/.relenv" PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" permissions: contents: write # To be able to publish the release diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index e9cbcaafd2f..bd1a41d4fb9 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -15,6 +15,7 @@ env: CACHE_SEED: SEED-7 # Bump the number to invalidate all caches RELENV_DATA: "${{ github.workspace }}/.relenv" PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" permissions: contents: read # for dorny/paths-filter to fetch a list of changed files diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 00b58b8ad6e..8618905eaa5 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -40,6 +40,7 @@ env: CACHE_SEED: SEED-7 # Bump the number to invalidate all caches RELENV_DATA: "${{ github.workspace }}/.relenv" PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" permissions: contents: read # for dorny/paths-filter to fetch a list of changed files diff --git a/.github/workflows/templates/build-deps-ci-action.yml.jinja b/.github/workflows/templates/build-deps-ci-action.yml.jinja index 1d3504f6720..a43f7b43d01 100644 --- a/.github/workflows/templates/build-deps-ci-action.yml.jinja +++ b/.github/workflows/templates/build-deps-ci-action.yml.jinja @@ -43,6 +43,7 @@ env: PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ PIP_EXTRA_INDEX_URL: https://pypi.org/simple PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" jobs: diff --git a/.github/workflows/templates/layout.yml.jinja b/.github/workflows/templates/layout.yml.jinja index 08f71095a8b..4b146ff3a5f 100644 --- a/.github/workflows/templates/layout.yml.jinja +++ b/.github/workflows/templates/layout.yml.jinja @@ -37,6 +37,7 @@ env: CACHE_SEED: SEED-7 # Bump the number to invalidate all caches RELENV_DATA: "${{ github.workspace }}/.relenv" PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" <%- endblock env %> diff --git a/.github/workflows/templates/test-package-downloads-action.yml.jinja b/.github/workflows/templates/test-package-downloads-action.yml.jinja index 8752263bbcb..6fb293cf971 100644 --- a/.github/workflows/templates/test-package-downloads-action.yml.jinja +++ b/.github/workflows/templates/test-package-downloads-action.yml.jinja @@ -51,6 +51,7 @@ env: PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ PIP_EXTRA_INDEX_URL: https://pypi.org/simple PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" jobs: diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index d7d1ae0b599..cca9cc029fe 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -74,6 +74,7 @@ env: PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ PIP_EXTRA_INDEX_URL: https://pypi.org/simple PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" jobs: diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index d9edaff242e..19cd30f2f3f 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -67,6 +67,7 @@ env: PIP_INDEX_URL: "https://pypi-proxy.saltstack.net/root/local/+simple/" PIP_EXTRA_INDEX_URL: "https://pypi.org/simple" PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" jobs: diff --git a/.github/workflows/test-action-windows.yml b/.github/workflows/test-action-windows.yml index 50cdcd4c8b4..4a7d496042c 100644 --- a/.github/workflows/test-action-windows.yml +++ b/.github/workflows/test-action-windows.yml @@ -74,6 +74,7 @@ env: PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ PIP_EXTRA_INDEX_URL: https://pypi.org/simple PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" jobs: diff --git a/.github/workflows/test-package-downloads-action.yml b/.github/workflows/test-package-downloads-action.yml index f746834120a..8f0f977a26c 100644 --- a/.github/workflows/test-package-downloads-action.yml +++ b/.github/workflows/test-package-downloads-action.yml @@ -51,6 +51,7 @@ env: PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ PIP_EXTRA_INDEX_URL: https://pypi.org/simple PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" jobs: diff --git a/.github/workflows/test-packages-action-linux.yml b/.github/workflows/test-packages-action-linux.yml index 355d9967b19..db8fe9b5efb 100644 --- a/.github/workflows/test-packages-action-linux.yml +++ b/.github/workflows/test-packages-action-linux.yml @@ -68,6 +68,7 @@ env: PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ PIP_EXTRA_INDEX_URL: https://pypi.org/simple PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" jobs: diff --git a/.github/workflows/test-packages-action-macos.yml b/.github/workflows/test-packages-action-macos.yml index 5d5b58dfc6e..0db11075663 100644 --- a/.github/workflows/test-packages-action-macos.yml +++ b/.github/workflows/test-packages-action-macos.yml @@ -61,6 +61,7 @@ env: PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ PIP_EXTRA_INDEX_URL: https://pypi.org/simple PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" jobs: diff --git a/.github/workflows/test-packages-action-windows.yml b/.github/workflows/test-packages-action-windows.yml index d0619857e05..bca0b852592 100644 --- a/.github/workflows/test-packages-action-windows.yml +++ b/.github/workflows/test-packages-action-windows.yml @@ -68,6 +68,7 @@ env: PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ PIP_EXTRA_INDEX_URL: https://pypi.org/simple PIP_DISABLE_PIP_VERSION_CHECK: "1" + RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" jobs: diff --git a/tools/vm.py b/tools/vm.py index 83493c3b04d..6a65c189f63 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -1438,6 +1438,7 @@ class VM: env["PYTHONUTF8"] = "1" env["OUTPUT_COLUMNS"] = str(self.ctx.console.width) env["GITHUB_ACTIONS_PIPELINE"] = "1" + env["RAISE_DEPRECATIONS_RUNTIME_ERRORS"] = "1" self.write_and_upload_dot_env(env) if self.is_windows is False and self.config.ssh_username != "root": sudo = True From 86cfde38ed06d537f4dd14f0e7ac3e4c88a07ce1 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 24 Jan 2024 15:05:46 +0000 Subject: [PATCH 15/29] Run `pyupgrade` against the files modified in the merge-forward --- salt/modules/ini_manage.py | 12 ++++++------ tests/pytests/unit/modules/test_ini_manage.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/modules/ini_manage.py b/salt/modules/ini_manage.py index 70ce447f5b0..f5aff8881e2 100644 --- a/salt/modules/ini_manage.py +++ b/salt/modules/ini_manage.py @@ -417,7 +417,7 @@ class _Section(OrderedDict): # Match comments com_match = COM_REGX.match(opt_str) if com_match: - name = "#comment{}".format(comment_count) + name = f"#comment{comment_count}" self.com = com_match.group(1) comment_count += 1 self.update({name: opt_str}) @@ -441,7 +441,7 @@ class _Section(OrderedDict): self.update({name: value}) continue # Anything remaining is a mystery. - name = "#unknown{}".format(unknown_count) + name = f"#unknown{unknown_count}" self.update({name: opt_str}) unknown_count += 1 @@ -512,7 +512,7 @@ class _Section(OrderedDict): for name, value in self.items(): # Handle Comment Lines if COM_REGX.match(name): - yield "{}{}".format(value, os.linesep) + yield f"{value}{os.linesep}" # Handle Sections elif isinstance(value, _Section): sections_dict.update({name: value}) @@ -521,7 +521,7 @@ class _Section(OrderedDict): else: yield "{}{}{}{}".format( name, - " {} ".format(self.sep) if self.sep != " " else self.sep, + f" {self.sep} " if self.sep != " " else self.sep, value, os.linesep, ) @@ -583,7 +583,7 @@ class _Ini(_Section): except OSError as exc: if __opts__["test"] is False: raise CommandExecutionError( - "Unable to open file '{}'. Exception: {}".format(self.name, exc) + f"Unable to open file '{self.name}'. Exception: {exc}" ) if not inicontents: return @@ -620,7 +620,7 @@ class _Ini(_Section): outfile.writelines(ini_gen_list) except OSError as exc: raise CommandExecutionError( - "Unable to write file '{}'. Exception: {}".format(self.name, exc) + f"Unable to write file '{self.name}'. Exception: {exc}" ) @staticmethod diff --git a/tests/pytests/unit/modules/test_ini_manage.py b/tests/pytests/unit/modules/test_ini_manage.py index 538e75c171b..27f8d46eac6 100644 --- a/tests/pytests/unit/modules/test_ini_manage.py +++ b/tests/pytests/unit/modules/test_ini_manage.py @@ -77,7 +77,7 @@ def test_section_req(): """ Test the __repr__ in the _Section class """ - expected = "_Section(){}{{}}".format(os.linesep) + expected = f"_Section(){os.linesep}{{}}" assert repr(ini._Section("test")) == expected From 98c92a3facb9326ad52b50925c6f5daf2e41ef04 Mon Sep 17 00:00:00 2001 From: Insoo Ha Date: Sun, 10 Dec 2023 18:50:07 +0900 Subject: [PATCH 16/29] Use send_multipart instead of send when sending multipart message. --- changelog/65018.fixed.md | 1 + salt/transport/zeromq.py | 9 +- .../zeromq/test_pub_server_channel.py | 193 ++++++++++++++++-- tests/support/pytest/transport.py | 45 +++- 4 files changed, 220 insertions(+), 28 deletions(-) create mode 100644 changelog/65018.fixed.md diff --git a/changelog/65018.fixed.md b/changelog/65018.fixed.md new file mode 100644 index 00000000000..24cc3388ed8 --- /dev/null +++ b/changelog/65018.fixed.md @@ -0,0 +1 @@ +Use `send_multipart` instead of `send` when sending multipart message. \ No newline at end of file diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index 4f54430cefd..07f2bc1bf89 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -777,21 +777,18 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer): htopic = salt.utils.stringutils.to_bytes( hashlib.sha1(salt.utils.stringutils.to_bytes(topic)).hexdigest() ) - yield self.dpub_sock.send(htopic, flags=zmq.SNDMORE) - yield self.dpub_sock.send(payload) + yield self.dpub_sock.send_multipart([htopic, payload]) log.trace("Filtered data has been sent") # Syndic broadcast if self.opts.get("order_masters"): log.trace("Sending filtered data to syndic") - yield self.dpub_sock.send(b"syndic", flags=zmq.SNDMORE) - yield self.dpub_sock.send(payload) + yield self.dpub_sock.send_multipart([b"syndic", payload]) log.trace("Filtered data has been sent to syndic") # otherwise its a broadcast else: # TODO: constants file for "broadcast" log.trace("Sending broadcasted data over publisher %s", self.pub_uri) - yield self.dpub_sock.send(b"broadcast", flags=zmq.SNDMORE) - yield self.dpub_sock.send(payload) + yield self.dpub_sock.send_multipart([b"broadcast", payload]) log.trace("Broadcasted data has been sent") else: log.trace("Sending ZMQ-unfiltered data over publisher %s", self.pub_uri) diff --git a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py index 2a357c7c5db..7ab1c069981 100644 --- a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py +++ b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py @@ -1,10 +1,16 @@ +from contextlib import contextmanager +import copy import logging -import threading +import random import time +import threading import pytest +from saltfactories.utils import random_string import salt.transport.zeromq +import salt.utils.process + from tests.support.mock import MagicMock, patch from tests.support.pytest.transport import PubServerChannelProcess @@ -20,9 +26,151 @@ pytestmark = [ ] +class PubServerChannelSender: + def __init__(self, pub_server_channel, payload_list): + self.pub_server_channel = pub_server_channel + self.payload_list = payload_list + + def run(self): + for payload in self.payload_list: + self.pub_server_channel.publish(payload) + time.sleep(2) + + +def generate_msg_list(msg_cnt, minions_list, broadcast): + msg_list = [] + for i in range(msg_cnt): + for idx, minion_id in enumerate(minions_list): + if broadcast: + msg_list.append({"tgt_type": "grain", "tgt": 'id:*', "jid": msg_cnt * idx + i}) + else: + msg_list.append({"tgt_type": "list", "tgt": [minion_id], "jid": msg_cnt * idx + i}) + return msg_list + + +@contextmanager +def channel_publisher_manager(msg_list, p_cnt, pub_server_channel): + process_list = [] + msg_list = copy.deepcopy(msg_list) + random.shuffle(msg_list) + batch_size = len(msg_list) // p_cnt + list_batch = [[x * batch_size, x * batch_size + batch_size] for x in range(0, p_cnt)] + list_batch[-1][1] = list_batch[-1][1] + 1 + try: + for i, j in list_batch: + c = PubServerChannelSender(pub_server_channel, msg_list[i:j]) + p = salt.utils.process.Process(target=c.run) + process_list.append(p) + for p in process_list: + p.start() + yield + finally: + for p in process_list: + p.join() + + @pytest.mark.skip_on_windows @pytest.mark.slow_test -def test_zeromq_filtering(salt_master, salt_minion): +def test_zeromq_filtering_minion(salt_master, salt_minion): + opts = dict( + salt_master.config.copy(), + ipc_mode="ipc", + pub_hwm=0, + zmq_filtering=True, + acceptance_wait_time=5, + ) + minion_opts = dict( + salt_minion.config.copy(), + zmq_filtering=True, + ) + messages = 200 + workers = 5 + minions = 3 + expect = set(range(messages)) + target_minion_id = salt_minion.id + minions_list = [target_minion_id] + for _ in range(minions - 1): + minions_list.append(random_string("zeromq-minion-")) + msg_list = generate_msg_list(messages, minions_list, False) + with patch( + "salt.utils.minions.CkMinions.check_minions", + MagicMock( + return_value={ + "minions": minions_list, + "missing": [], + "ssh_minions": False, + } + ), + ): + with PubServerChannelProcess(opts, minion_opts) as server_channel: + with channel_publisher_manager(msg_list, workers, server_channel.pub_server_channel): + cnt = 0 + last_results_len = 0 + while cnt < 20: + time.sleep(2) + results_len = len(server_channel.collector.results) + if last_results_len == results_len: + break + last_results_len = results_len + cnt += 1 + results = set(server_channel.collector.results) + assert results == expect, \ + f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" + + +@pytest.mark.skip_on_windows +@pytest.mark.slow_test +def test_zeromq_filtering_syndic(salt_master, salt_minion): + opts = dict( + salt_master.config.copy(), + ipc_mode="ipc", + pub_hwm=0, + zmq_filtering=True, + acceptance_wait_time=5, + order_masters=True, + ) + minion_opts = dict( + salt_minion.config.copy(), + zmq_filtering=True, + __role='syndic', + ) + messages = 200 + workers = 5 + minions = 3 + expect = set(range(messages * minions)) + minions_list = [] + for _ in range(minions): + minions_list.append(random_string("zeromq-minion-")) + msg_list = generate_msg_list(messages, minions_list, False) + with patch( + "salt.utils.minions.CkMinions.check_minions", + MagicMock( + return_value={ + "minions": minions_list, + "missing": [], + "ssh_minions": False, + } + ), + ): + with PubServerChannelProcess(opts, minion_opts) as server_channel: + with channel_publisher_manager(msg_list, workers, server_channel.pub_server_channel): + cnt = 0 + last_results_len = 0 + while cnt < 20: + time.sleep(2) + results_len = len(server_channel.collector.results) + if last_results_len == results_len: + break + last_results_len = results_len + cnt += 1 + results = set(server_channel.collector.results) + assert results == expect, \ + f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" + + +@pytest.mark.skip_on_windows +@pytest.mark.slow_test +def test_zeromq_filtering_broadcast(salt_master, salt_minion): """ Test sending messages to publisher using UDP with zeromq_filtering enabled """ @@ -33,28 +181,43 @@ def test_zeromq_filtering(salt_master, salt_minion): zmq_filtering=True, acceptance_wait_time=5, ) - send_num = 1 - expect = [] + minion_opts = dict( + salt_minion.config.copy(), + zmq_filtering=True, + ) + messages = 200 + workers = 5 + minions = 3 + expect = set(range(messages * minions)) + target_minion_id = salt_minion.id + minions_list = [target_minion_id] + for _ in range(minions - 1): + minions_list.append(random_string("zeromq-minion-")) + msg_list = generate_msg_list(messages, minions_list, True) with patch( "salt.utils.minions.CkMinions.check_minions", MagicMock( return_value={ - "minions": [salt_minion.id], + "minions": minions_list, "missing": [], "ssh_minions": False, } ), ): - with PubServerChannelProcess( - opts, salt_minion.config.copy(), zmq_filtering=True - ) as server_channel: - expect.append(send_num) - load = {"tgt_type": "glob", "tgt": "*", "jid": send_num} - server_channel.publish(load) - results = server_channel.collector.results - assert len(results) == send_num, "{} != {}, difference: {}".format( - len(results), send_num, set(expect).difference(results) - ) + with PubServerChannelProcess(opts, minion_opts) as server_channel: + with channel_publisher_manager(msg_list, workers, server_channel.pub_server_channel): + cnt = 0 + last_results_len = 0 + while cnt < 20: + time.sleep(2) + results_len = len(server_channel.collector.results) + if last_results_len == results_len: + break + last_results_len = results_len + cnt += 1 + results = set(server_channel.collector.results) + assert results == expect, \ + f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" def test_pub_channel(master_opts): diff --git a/tests/support/pytest/transport.py b/tests/support/pytest/transport.py index d71e8fe0254..2b677d8475d 100644 --- a/tests/support/pytest/transport.py +++ b/tests/support/pytest/transport.py @@ -1,4 +1,5 @@ import ctypes +import hashlib import logging import multiprocessing import socket @@ -9,11 +10,13 @@ import zmq from pytestshellutils.utils.processes import terminate_process import salt.channel.server +import salt.crypt import salt.exceptions import salt.ext.tornado.gen import salt.ext.tornado.ioloop import salt.ext.tornado.iostream import salt.master +import salt.payload import salt.utils.msgpack import salt.utils.process import salt.utils.stringutils @@ -36,10 +39,10 @@ class Collector(salt.utils.process.SignalHandlingProcess): port, aes_key, timeout=300, - zmq_filtering=False, ): super().__init__() self.minion_config = minion_config + self.hexid = hashlib.sha1(salt.utils.stringutils.to_bytes(self.minion_config["id"])).hexdigest() self.interface = interface self.port = port self.aes_key = aes_key @@ -48,10 +51,11 @@ class Collector(salt.utils.process.SignalHandlingProcess): self.hard_timeout = time.time() + timeout + 120 self.manager = multiprocessing.Manager() self.results = self.manager.list() - self.zmq_filtering = zmq_filtering + self.zmq_filtering = minion_config['zmq_filtering'] self.stopped = multiprocessing.Event() self.started = multiprocessing.Event() self.running = multiprocessing.Event() + self.stop_running = multiprocessing.Event() self.unpacker = salt.utils.msgpack.Unpacker(raw=False) @property @@ -78,7 +82,14 @@ class Collector(salt.utils.process.SignalHandlingProcess): ctx = zmq.Context() self.sock = ctx.socket(zmq.SUB) self.sock.setsockopt(zmq.LINGER, -1) - self.sock.setsockopt(zmq.SUBSCRIBE, b"") + if self.zmq_filtering: + self.sock.setsockopt(zmq.SUBSCRIBE, b"broadcast") + if self.minion_config.get("__role") == "syndic": + self.sock.setsockopt(zmq.SUBSCRIBE, b"syndic") + else: + self.sock.setsockopt(zmq.SUBSCRIBE, salt.utils.stringutils.to_bytes(self.hexid)) + else: + self.sock.setsockopt(zmq.SUBSCRIBE, b"") pub_uri = "tcp://{}:{}".format(self.interface, self.port) self.sock.connect(pub_uri) else: @@ -101,8 +112,23 @@ class Collector(salt.utils.process.SignalHandlingProcess): # test_zeromq_filtering requires catching the # SaltDeserializationError in order to pass. try: - payload = self.sock.recv(zmq.NOBLOCK) - serial_payload = salt.payload.loads(payload) + messages = self.sock.recv_multipart(zmq.NOBLOCK) + messages_len = len(messages) + if messages_len == 1: + serial_payload = salt.payload.loads(messages[0]) + elif messages_len == 2: + message_target = salt.utils.stringutils.to_str(messages[0]) + is_syndic = self.minion_config.get("__role") == "syndic" + if ( + not is_syndic and message_target not in ("broadcast", self.hexid) + ) or ( + is_syndic and message_target not in ("broadcast", "syndic") + ): + log.debug("Publish received for not this minion: %s", message_target) + raise salt.ext.tornado.gen.Return(None) + serial_payload = salt.payload.loads(messages[1]) + else: + raise Exception("Invalid number of messages") raise salt.ext.tornado.gen.Return(serial_payload) except (zmq.ZMQError, salt.exceptions.SaltDeserializationError): raise RecvError("ZMQ Error") @@ -125,7 +151,6 @@ class Collector(salt.utils.process.SignalHandlingProcess): return self.started.set() last_msg = time.time() - serial = salt.payload.Serial(self.minion_config) crypticle = salt.crypt.Crypticle(self.minion_config, self.aes_key) while True: curr_time = time.time() @@ -150,6 +175,7 @@ class Collector(salt.utils.process.SignalHandlingProcess): continue if "stop" in payload: log.info("Collector stopped") + self.stop_running.set() break last_msg = time.time() self.results.append(payload["jid"]) @@ -275,7 +301,12 @@ class PubServerChannelProcess(salt.utils.process.SignalHandlingProcess): def __exit__(self, *args): # Publish a payload to tell the collection it's done processing - self.publish({"tgt_type": "glob", "tgt": "*", "jid": -1, "stop": True}) + attempts = 300 + while attempts > 0: + self.publish({"tgt_type": "glob", "tgt": "*", "jid": -1, "stop": True}) + if self.collector.stop_running.wait(1) is True: + break + attempts -= 1 # Now trigger the collector to also exit self.collector.__exit__(*args) # We can safely wait here without a timeout because the Collector instance has a From 95bccf06c2d8578c4bfdb4f495aede11b7ac6f9c Mon Sep 17 00:00:00 2001 From: Insoo Ha Date: Thu, 14 Dec 2023 09:56:20 +0900 Subject: [PATCH 17/29] fix pre-commit --- changelog/65018.fixed.md | 2 +- requirements/static/ci/py3.8/lint.txt | 10 ---- .../zeromq/test_pub_server_channel.py | 48 ++++++++++++------- tests/support/pytest/transport.py | 21 ++++---- 4 files changed, 45 insertions(+), 36 deletions(-) diff --git a/changelog/65018.fixed.md b/changelog/65018.fixed.md index 24cc3388ed8..c719aac9f99 100644 --- a/changelog/65018.fixed.md +++ b/changelog/65018.fixed.md @@ -1 +1 @@ -Use `send_multipart` instead of `send` when sending multipart message. \ No newline at end of file +Use `send_multipart` instead of `send` when sending multipart message. diff --git a/requirements/static/ci/py3.8/lint.txt b/requirements/static/ci/py3.8/lint.txt index 5b10cbe8a36..64bf8f6f12a 100644 --- a/requirements/static/ci/py3.8/lint.txt +++ b/requirements/static/ci/py3.8/lint.txt @@ -69,10 +69,6 @@ cachetools==4.2.2 # -c requirements/static/ci/py3.8/linux.txt # google-auth # python-telegram-bot -cassandra-driver==3.23.0 - # via - # -c requirements/static/ci/py3.8/linux.txt - # -r requirements/static/ci/common.in certifi==2023.07.22 # via # -c requirements/static/ci/../pkg/py3.8/linux.txt @@ -307,12 +303,6 @@ looseversion==1.0.2 # -c requirements/static/ci/../pkg/py3.8/linux.txt # -c requirements/static/ci/py3.8/linux.txt # -r requirements/base.txt -lxml==4.9.1 - # via - # -c requirements/static/ci/py3.8/linux.txt - # junos-eznc - # napalm - # ncclient mako==1.2.2 # via # -c requirements/static/ci/py3.8/linux.txt diff --git a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py index 7ab1c069981..4e851a52fd3 100644 --- a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py +++ b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py @@ -1,16 +1,15 @@ -from contextlib import contextmanager import copy import logging import random -import time import threading +import time +from contextlib import contextmanager import pytest - from saltfactories.utils import random_string + import salt.transport.zeromq import salt.utils.process - from tests.support.mock import MagicMock, patch from tests.support.pytest.transport import PubServerChannelProcess @@ -42,9 +41,13 @@ def generate_msg_list(msg_cnt, minions_list, broadcast): for i in range(msg_cnt): for idx, minion_id in enumerate(minions_list): if broadcast: - msg_list.append({"tgt_type": "grain", "tgt": 'id:*', "jid": msg_cnt * idx + i}) + msg_list.append( + {"tgt_type": "grain", "tgt": "id:*", "jid": msg_cnt * idx + i} + ) else: - msg_list.append({"tgt_type": "list", "tgt": [minion_id], "jid": msg_cnt * idx + i}) + msg_list.append( + {"tgt_type": "list", "tgt": [minion_id], "jid": msg_cnt * idx + i} + ) return msg_list @@ -54,7 +57,9 @@ def channel_publisher_manager(msg_list, p_cnt, pub_server_channel): msg_list = copy.deepcopy(msg_list) random.shuffle(msg_list) batch_size = len(msg_list) // p_cnt - list_batch = [[x * batch_size, x * batch_size + batch_size] for x in range(0, p_cnt)] + list_batch = [ + [x * batch_size, x * batch_size + batch_size] for x in range(0, p_cnt) + ] list_batch[-1][1] = list_batch[-1][1] + 1 try: for i, j in list_batch: @@ -103,7 +108,9 @@ def test_zeromq_filtering_minion(salt_master, salt_minion): ), ): with PubServerChannelProcess(opts, minion_opts) as server_channel: - with channel_publisher_manager(msg_list, workers, server_channel.pub_server_channel): + with channel_publisher_manager( + msg_list, workers, server_channel.pub_server_channel + ): cnt = 0 last_results_len = 0 while cnt < 20: @@ -114,8 +121,9 @@ def test_zeromq_filtering_minion(salt_master, salt_minion): last_results_len = results_len cnt += 1 results = set(server_channel.collector.results) - assert results == expect, \ - f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" + assert ( + results == expect + ), f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" @pytest.mark.skip_on_windows @@ -132,7 +140,7 @@ def test_zeromq_filtering_syndic(salt_master, salt_minion): minion_opts = dict( salt_minion.config.copy(), zmq_filtering=True, - __role='syndic', + __role="syndic", ) messages = 200 workers = 5 @@ -153,7 +161,9 @@ def test_zeromq_filtering_syndic(salt_master, salt_minion): ), ): with PubServerChannelProcess(opts, minion_opts) as server_channel: - with channel_publisher_manager(msg_list, workers, server_channel.pub_server_channel): + with channel_publisher_manager( + msg_list, workers, server_channel.pub_server_channel + ): cnt = 0 last_results_len = 0 while cnt < 20: @@ -164,8 +174,9 @@ def test_zeromq_filtering_syndic(salt_master, salt_minion): last_results_len = results_len cnt += 1 results = set(server_channel.collector.results) - assert results == expect, \ - f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" + assert ( + results == expect + ), f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" @pytest.mark.skip_on_windows @@ -205,7 +216,9 @@ def test_zeromq_filtering_broadcast(salt_master, salt_minion): ), ): with PubServerChannelProcess(opts, minion_opts) as server_channel: - with channel_publisher_manager(msg_list, workers, server_channel.pub_server_channel): + with channel_publisher_manager( + msg_list, workers, server_channel.pub_server_channel + ): cnt = 0 last_results_len = 0 while cnt < 20: @@ -216,8 +229,9 @@ def test_zeromq_filtering_broadcast(salt_master, salt_minion): last_results_len = results_len cnt += 1 results = set(server_channel.collector.results) - assert results == expect, \ - f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" + assert ( + results == expect + ), f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" def test_pub_channel(master_opts): diff --git a/tests/support/pytest/transport.py b/tests/support/pytest/transport.py index 2b677d8475d..7c59b394351 100644 --- a/tests/support/pytest/transport.py +++ b/tests/support/pytest/transport.py @@ -42,7 +42,9 @@ class Collector(salt.utils.process.SignalHandlingProcess): ): super().__init__() self.minion_config = minion_config - self.hexid = hashlib.sha1(salt.utils.stringutils.to_bytes(self.minion_config["id"])).hexdigest() + self.hexid = hashlib.sha1( + salt.utils.stringutils.to_bytes(self.minion_config["id"]) + ).hexdigest() self.interface = interface self.port = port self.aes_key = aes_key @@ -51,7 +53,7 @@ class Collector(salt.utils.process.SignalHandlingProcess): self.hard_timeout = time.time() + timeout + 120 self.manager = multiprocessing.Manager() self.results = self.manager.list() - self.zmq_filtering = minion_config['zmq_filtering'] + self.zmq_filtering = minion_config["zmq_filtering"] self.stopped = multiprocessing.Event() self.started = multiprocessing.Event() self.running = multiprocessing.Event() @@ -87,7 +89,9 @@ class Collector(salt.utils.process.SignalHandlingProcess): if self.minion_config.get("__role") == "syndic": self.sock.setsockopt(zmq.SUBSCRIBE, b"syndic") else: - self.sock.setsockopt(zmq.SUBSCRIBE, salt.utils.stringutils.to_bytes(self.hexid)) + self.sock.setsockopt( + zmq.SUBSCRIBE, salt.utils.stringutils.to_bytes(self.hexid) + ) else: self.sock.setsockopt(zmq.SUBSCRIBE, b"") pub_uri = "tcp://{}:{}".format(self.interface, self.port) @@ -120,11 +124,12 @@ class Collector(salt.utils.process.SignalHandlingProcess): message_target = salt.utils.stringutils.to_str(messages[0]) is_syndic = self.minion_config.get("__role") == "syndic" if ( - not is_syndic and message_target not in ("broadcast", self.hexid) - ) or ( - is_syndic and message_target not in ("broadcast", "syndic") - ): - log.debug("Publish received for not this minion: %s", message_target) + not is_syndic + and message_target not in ("broadcast", self.hexid) + ) or (is_syndic and message_target not in ("broadcast", "syndic")): + log.debug( + "Publish received for not this minion: %s", message_target + ) raise salt.ext.tornado.gen.Return(None) serial_payload = salt.payload.loads(messages[1]) else: From 13cfa9341f0c2eaae4e0e7a215cc7104bf9397a6 Mon Sep 17 00:00:00 2001 From: Insoo Ha Date: Thu, 14 Dec 2023 10:02:52 +0900 Subject: [PATCH 18/29] Revert "fix pre-commit" This reverts commit 9f779c68cfef5683fc8ffc2cf7f39a94366a4fe1. --- changelog/65018.fixed.md | 2 +- requirements/static/ci/py3.8/lint.txt | 10 ++++ .../zeromq/test_pub_server_channel.py | 48 +++++++------------ tests/support/pytest/transport.py | 21 ++++---- 4 files changed, 36 insertions(+), 45 deletions(-) diff --git a/changelog/65018.fixed.md b/changelog/65018.fixed.md index c719aac9f99..24cc3388ed8 100644 --- a/changelog/65018.fixed.md +++ b/changelog/65018.fixed.md @@ -1 +1 @@ -Use `send_multipart` instead of `send` when sending multipart message. +Use `send_multipart` instead of `send` when sending multipart message. \ No newline at end of file diff --git a/requirements/static/ci/py3.8/lint.txt b/requirements/static/ci/py3.8/lint.txt index 64bf8f6f12a..5b10cbe8a36 100644 --- a/requirements/static/ci/py3.8/lint.txt +++ b/requirements/static/ci/py3.8/lint.txt @@ -69,6 +69,10 @@ cachetools==4.2.2 # -c requirements/static/ci/py3.8/linux.txt # google-auth # python-telegram-bot +cassandra-driver==3.23.0 + # via + # -c requirements/static/ci/py3.8/linux.txt + # -r requirements/static/ci/common.in certifi==2023.07.22 # via # -c requirements/static/ci/../pkg/py3.8/linux.txt @@ -303,6 +307,12 @@ looseversion==1.0.2 # -c requirements/static/ci/../pkg/py3.8/linux.txt # -c requirements/static/ci/py3.8/linux.txt # -r requirements/base.txt +lxml==4.9.1 + # via + # -c requirements/static/ci/py3.8/linux.txt + # junos-eznc + # napalm + # ncclient mako==1.2.2 # via # -c requirements/static/ci/py3.8/linux.txt diff --git a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py index 4e851a52fd3..7ab1c069981 100644 --- a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py +++ b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py @@ -1,15 +1,16 @@ +from contextlib import contextmanager import copy import logging import random -import threading import time -from contextlib import contextmanager +import threading import pytest -from saltfactories.utils import random_string +from saltfactories.utils import random_string import salt.transport.zeromq import salt.utils.process + from tests.support.mock import MagicMock, patch from tests.support.pytest.transport import PubServerChannelProcess @@ -41,13 +42,9 @@ def generate_msg_list(msg_cnt, minions_list, broadcast): for i in range(msg_cnt): for idx, minion_id in enumerate(minions_list): if broadcast: - msg_list.append( - {"tgt_type": "grain", "tgt": "id:*", "jid": msg_cnt * idx + i} - ) + msg_list.append({"tgt_type": "grain", "tgt": 'id:*', "jid": msg_cnt * idx + i}) else: - msg_list.append( - {"tgt_type": "list", "tgt": [minion_id], "jid": msg_cnt * idx + i} - ) + msg_list.append({"tgt_type": "list", "tgt": [minion_id], "jid": msg_cnt * idx + i}) return msg_list @@ -57,9 +54,7 @@ def channel_publisher_manager(msg_list, p_cnt, pub_server_channel): msg_list = copy.deepcopy(msg_list) random.shuffle(msg_list) batch_size = len(msg_list) // p_cnt - list_batch = [ - [x * batch_size, x * batch_size + batch_size] for x in range(0, p_cnt) - ] + list_batch = [[x * batch_size, x * batch_size + batch_size] for x in range(0, p_cnt)] list_batch[-1][1] = list_batch[-1][1] + 1 try: for i, j in list_batch: @@ -108,9 +103,7 @@ def test_zeromq_filtering_minion(salt_master, salt_minion): ), ): with PubServerChannelProcess(opts, minion_opts) as server_channel: - with channel_publisher_manager( - msg_list, workers, server_channel.pub_server_channel - ): + with channel_publisher_manager(msg_list, workers, server_channel.pub_server_channel): cnt = 0 last_results_len = 0 while cnt < 20: @@ -121,9 +114,8 @@ def test_zeromq_filtering_minion(salt_master, salt_minion): last_results_len = results_len cnt += 1 results = set(server_channel.collector.results) - assert ( - results == expect - ), f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" + assert results == expect, \ + f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" @pytest.mark.skip_on_windows @@ -140,7 +132,7 @@ def test_zeromq_filtering_syndic(salt_master, salt_minion): minion_opts = dict( salt_minion.config.copy(), zmq_filtering=True, - __role="syndic", + __role='syndic', ) messages = 200 workers = 5 @@ -161,9 +153,7 @@ def test_zeromq_filtering_syndic(salt_master, salt_minion): ), ): with PubServerChannelProcess(opts, minion_opts) as server_channel: - with channel_publisher_manager( - msg_list, workers, server_channel.pub_server_channel - ): + with channel_publisher_manager(msg_list, workers, server_channel.pub_server_channel): cnt = 0 last_results_len = 0 while cnt < 20: @@ -174,9 +164,8 @@ def test_zeromq_filtering_syndic(salt_master, salt_minion): last_results_len = results_len cnt += 1 results = set(server_channel.collector.results) - assert ( - results == expect - ), f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" + assert results == expect, \ + f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" @pytest.mark.skip_on_windows @@ -216,9 +205,7 @@ def test_zeromq_filtering_broadcast(salt_master, salt_minion): ), ): with PubServerChannelProcess(opts, minion_opts) as server_channel: - with channel_publisher_manager( - msg_list, workers, server_channel.pub_server_channel - ): + with channel_publisher_manager(msg_list, workers, server_channel.pub_server_channel): cnt = 0 last_results_len = 0 while cnt < 20: @@ -229,9 +216,8 @@ def test_zeromq_filtering_broadcast(salt_master, salt_minion): last_results_len = results_len cnt += 1 results = set(server_channel.collector.results) - assert ( - results == expect - ), f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" + assert results == expect, \ + f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" def test_pub_channel(master_opts): diff --git a/tests/support/pytest/transport.py b/tests/support/pytest/transport.py index 7c59b394351..2b677d8475d 100644 --- a/tests/support/pytest/transport.py +++ b/tests/support/pytest/transport.py @@ -42,9 +42,7 @@ class Collector(salt.utils.process.SignalHandlingProcess): ): super().__init__() self.minion_config = minion_config - self.hexid = hashlib.sha1( - salt.utils.stringutils.to_bytes(self.minion_config["id"]) - ).hexdigest() + self.hexid = hashlib.sha1(salt.utils.stringutils.to_bytes(self.minion_config["id"])).hexdigest() self.interface = interface self.port = port self.aes_key = aes_key @@ -53,7 +51,7 @@ class Collector(salt.utils.process.SignalHandlingProcess): self.hard_timeout = time.time() + timeout + 120 self.manager = multiprocessing.Manager() self.results = self.manager.list() - self.zmq_filtering = minion_config["zmq_filtering"] + self.zmq_filtering = minion_config['zmq_filtering'] self.stopped = multiprocessing.Event() self.started = multiprocessing.Event() self.running = multiprocessing.Event() @@ -89,9 +87,7 @@ class Collector(salt.utils.process.SignalHandlingProcess): if self.minion_config.get("__role") == "syndic": self.sock.setsockopt(zmq.SUBSCRIBE, b"syndic") else: - self.sock.setsockopt( - zmq.SUBSCRIBE, salt.utils.stringutils.to_bytes(self.hexid) - ) + self.sock.setsockopt(zmq.SUBSCRIBE, salt.utils.stringutils.to_bytes(self.hexid)) else: self.sock.setsockopt(zmq.SUBSCRIBE, b"") pub_uri = "tcp://{}:{}".format(self.interface, self.port) @@ -124,12 +120,11 @@ class Collector(salt.utils.process.SignalHandlingProcess): message_target = salt.utils.stringutils.to_str(messages[0]) is_syndic = self.minion_config.get("__role") == "syndic" if ( - not is_syndic - and message_target not in ("broadcast", self.hexid) - ) or (is_syndic and message_target not in ("broadcast", "syndic")): - log.debug( - "Publish received for not this minion: %s", message_target - ) + not is_syndic and message_target not in ("broadcast", self.hexid) + ) or ( + is_syndic and message_target not in ("broadcast", "syndic") + ): + log.debug("Publish received for not this minion: %s", message_target) raise salt.ext.tornado.gen.Return(None) serial_payload = salt.payload.loads(messages[1]) else: From 10b9aeda05602c52acb07ce201cce3610db58c39 Mon Sep 17 00:00:00 2001 From: Insoo Ha Date: Thu, 14 Dec 2023 10:04:21 +0900 Subject: [PATCH 19/29] fix pre-commit --- changelog/65018.fixed.md | 2 +- .../zeromq/test_pub_server_channel.py | 48 ++++++++++++------- tests/support/pytest/transport.py | 21 ++++---- 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/changelog/65018.fixed.md b/changelog/65018.fixed.md index 24cc3388ed8..c719aac9f99 100644 --- a/changelog/65018.fixed.md +++ b/changelog/65018.fixed.md @@ -1 +1 @@ -Use `send_multipart` instead of `send` when sending multipart message. \ No newline at end of file +Use `send_multipart` instead of `send` when sending multipart message. diff --git a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py index 7ab1c069981..4e851a52fd3 100644 --- a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py +++ b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py @@ -1,16 +1,15 @@ -from contextlib import contextmanager import copy import logging import random -import time import threading +import time +from contextlib import contextmanager import pytest - from saltfactories.utils import random_string + import salt.transport.zeromq import salt.utils.process - from tests.support.mock import MagicMock, patch from tests.support.pytest.transport import PubServerChannelProcess @@ -42,9 +41,13 @@ def generate_msg_list(msg_cnt, minions_list, broadcast): for i in range(msg_cnt): for idx, minion_id in enumerate(minions_list): if broadcast: - msg_list.append({"tgt_type": "grain", "tgt": 'id:*', "jid": msg_cnt * idx + i}) + msg_list.append( + {"tgt_type": "grain", "tgt": "id:*", "jid": msg_cnt * idx + i} + ) else: - msg_list.append({"tgt_type": "list", "tgt": [minion_id], "jid": msg_cnt * idx + i}) + msg_list.append( + {"tgt_type": "list", "tgt": [minion_id], "jid": msg_cnt * idx + i} + ) return msg_list @@ -54,7 +57,9 @@ def channel_publisher_manager(msg_list, p_cnt, pub_server_channel): msg_list = copy.deepcopy(msg_list) random.shuffle(msg_list) batch_size = len(msg_list) // p_cnt - list_batch = [[x * batch_size, x * batch_size + batch_size] for x in range(0, p_cnt)] + list_batch = [ + [x * batch_size, x * batch_size + batch_size] for x in range(0, p_cnt) + ] list_batch[-1][1] = list_batch[-1][1] + 1 try: for i, j in list_batch: @@ -103,7 +108,9 @@ def test_zeromq_filtering_minion(salt_master, salt_minion): ), ): with PubServerChannelProcess(opts, minion_opts) as server_channel: - with channel_publisher_manager(msg_list, workers, server_channel.pub_server_channel): + with channel_publisher_manager( + msg_list, workers, server_channel.pub_server_channel + ): cnt = 0 last_results_len = 0 while cnt < 20: @@ -114,8 +121,9 @@ def test_zeromq_filtering_minion(salt_master, salt_minion): last_results_len = results_len cnt += 1 results = set(server_channel.collector.results) - assert results == expect, \ - f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" + assert ( + results == expect + ), f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" @pytest.mark.skip_on_windows @@ -132,7 +140,7 @@ def test_zeromq_filtering_syndic(salt_master, salt_minion): minion_opts = dict( salt_minion.config.copy(), zmq_filtering=True, - __role='syndic', + __role="syndic", ) messages = 200 workers = 5 @@ -153,7 +161,9 @@ def test_zeromq_filtering_syndic(salt_master, salt_minion): ), ): with PubServerChannelProcess(opts, minion_opts) as server_channel: - with channel_publisher_manager(msg_list, workers, server_channel.pub_server_channel): + with channel_publisher_manager( + msg_list, workers, server_channel.pub_server_channel + ): cnt = 0 last_results_len = 0 while cnt < 20: @@ -164,8 +174,9 @@ def test_zeromq_filtering_syndic(salt_master, salt_minion): last_results_len = results_len cnt += 1 results = set(server_channel.collector.results) - assert results == expect, \ - f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" + assert ( + results == expect + ), f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" @pytest.mark.skip_on_windows @@ -205,7 +216,9 @@ def test_zeromq_filtering_broadcast(salt_master, salt_minion): ), ): with PubServerChannelProcess(opts, minion_opts) as server_channel: - with channel_publisher_manager(msg_list, workers, server_channel.pub_server_channel): + with channel_publisher_manager( + msg_list, workers, server_channel.pub_server_channel + ): cnt = 0 last_results_len = 0 while cnt < 20: @@ -216,8 +229,9 @@ def test_zeromq_filtering_broadcast(salt_master, salt_minion): last_results_len = results_len cnt += 1 results = set(server_channel.collector.results) - assert results == expect, \ - f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" + assert ( + results == expect + ), f"{len(results)}, != {len(expect)}, difference: {expect.difference(results)} {results}" def test_pub_channel(master_opts): diff --git a/tests/support/pytest/transport.py b/tests/support/pytest/transport.py index 2b677d8475d..7c59b394351 100644 --- a/tests/support/pytest/transport.py +++ b/tests/support/pytest/transport.py @@ -42,7 +42,9 @@ class Collector(salt.utils.process.SignalHandlingProcess): ): super().__init__() self.minion_config = minion_config - self.hexid = hashlib.sha1(salt.utils.stringutils.to_bytes(self.minion_config["id"])).hexdigest() + self.hexid = hashlib.sha1( + salt.utils.stringutils.to_bytes(self.minion_config["id"]) + ).hexdigest() self.interface = interface self.port = port self.aes_key = aes_key @@ -51,7 +53,7 @@ class Collector(salt.utils.process.SignalHandlingProcess): self.hard_timeout = time.time() + timeout + 120 self.manager = multiprocessing.Manager() self.results = self.manager.list() - self.zmq_filtering = minion_config['zmq_filtering'] + self.zmq_filtering = minion_config["zmq_filtering"] self.stopped = multiprocessing.Event() self.started = multiprocessing.Event() self.running = multiprocessing.Event() @@ -87,7 +89,9 @@ class Collector(salt.utils.process.SignalHandlingProcess): if self.minion_config.get("__role") == "syndic": self.sock.setsockopt(zmq.SUBSCRIBE, b"syndic") else: - self.sock.setsockopt(zmq.SUBSCRIBE, salt.utils.stringutils.to_bytes(self.hexid)) + self.sock.setsockopt( + zmq.SUBSCRIBE, salt.utils.stringutils.to_bytes(self.hexid) + ) else: self.sock.setsockopt(zmq.SUBSCRIBE, b"") pub_uri = "tcp://{}:{}".format(self.interface, self.port) @@ -120,11 +124,12 @@ class Collector(salt.utils.process.SignalHandlingProcess): message_target = salt.utils.stringutils.to_str(messages[0]) is_syndic = self.minion_config.get("__role") == "syndic" if ( - not is_syndic and message_target not in ("broadcast", self.hexid) - ) or ( - is_syndic and message_target not in ("broadcast", "syndic") - ): - log.debug("Publish received for not this minion: %s", message_target) + not is_syndic + and message_target not in ("broadcast", self.hexid) + ) or (is_syndic and message_target not in ("broadcast", "syndic")): + log.debug( + "Publish received for not this minion: %s", message_target + ) raise salt.ext.tornado.gen.Return(None) serial_payload = salt.payload.loads(messages[1]) else: From 4708fe8900741acffcf6b99b243bb392fd0f6cc3 Mon Sep 17 00:00:00 2001 From: Insoo Ha Date: Sun, 17 Dec 2023 20:58:55 +0900 Subject: [PATCH 20/29] Fix missing error handling. --- tests/support/pytest/transport.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/support/pytest/transport.py b/tests/support/pytest/transport.py index 7c59b394351..0e30e44c8c4 100644 --- a/tests/support/pytest/transport.py +++ b/tests/support/pytest/transport.py @@ -312,6 +312,8 @@ class PubServerChannelProcess(salt.utils.process.SignalHandlingProcess): if self.collector.stop_running.wait(1) is True: break attempts -= 1 + else: + pytest.fail("Failed to confirm the collector has stopped") # Now trigger the collector to also exit self.collector.__exit__(*args) # We can safely wait here without a timeout because the Collector instance has a From a39553446976451b73dcb4f06dedd677281de822 Mon Sep 17 00:00:00 2001 From: Insoo Ha Date: Mon, 8 Jan 2024 17:57:10 +0900 Subject: [PATCH 21/29] Resolve lint failed --- tests/support/pytest/transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/support/pytest/transport.py b/tests/support/pytest/transport.py index 0e30e44c8c4..93e3418a39d 100644 --- a/tests/support/pytest/transport.py +++ b/tests/support/pytest/transport.py @@ -116,7 +116,7 @@ class Collector(salt.utils.process.SignalHandlingProcess): # test_zeromq_filtering requires catching the # SaltDeserializationError in order to pass. try: - messages = self.sock.recv_multipart(zmq.NOBLOCK) + messages = self.sock.recv_multipart(flags=zmq.NOBLOCK, copy=True) messages_len = len(messages) if messages_len == 1: serial_payload = salt.payload.loads(messages[0]) From 02ea8d48da15768cdd272f2b04c2adeaebe6f38d Mon Sep 17 00:00:00 2001 From: Twangboy Date: Mon, 22 Jan 2024 15:41:25 -0700 Subject: [PATCH 22/29] Add xmltodict to Windows requirements --- requirements/static/pkg/py3.10/windows.txt | 2 ++ requirements/static/pkg/py3.11/windows.txt | 2 ++ requirements/static/pkg/py3.12/windows.txt | 2 ++ requirements/static/pkg/py3.7/windows.txt | 2 ++ requirements/static/pkg/py3.8/windows.txt | 2 ++ requirements/static/pkg/py3.9/windows.txt | 2 ++ requirements/windows.txt | 1 + 7 files changed, 13 insertions(+) diff --git a/requirements/static/pkg/py3.10/windows.txt b/requirements/static/pkg/py3.10/windows.txt index a231b0bdb08..91651095db3 100644 --- a/requirements/static/pkg/py3.10/windows.txt +++ b/requirements/static/pkg/py3.10/windows.txt @@ -132,6 +132,8 @@ wheel==0.38.4 # via -r requirements/windows.txt wmi==1.5.1 # via -r requirements/windows.txt +xmltodict==0.13.0 + # via -r requirements/windows.txt zc.lockfile==2.0 # via cherrypy zipp==3.12.0 diff --git a/requirements/static/pkg/py3.11/windows.txt b/requirements/static/pkg/py3.11/windows.txt index be0822d71e5..8e4342a7d0c 100644 --- a/requirements/static/pkg/py3.11/windows.txt +++ b/requirements/static/pkg/py3.11/windows.txt @@ -132,6 +132,8 @@ wheel==0.38.4 # via -r requirements/windows.txt wmi==1.5.1 # via -r requirements/windows.txt +xmltodict==0.13.0 + # via -r requirements/windows.txt zc.lockfile==2.0 # via cherrypy zipp==3.12.0 diff --git a/requirements/static/pkg/py3.12/windows.txt b/requirements/static/pkg/py3.12/windows.txt index 15fff737182..cc7ba269db1 100644 --- a/requirements/static/pkg/py3.12/windows.txt +++ b/requirements/static/pkg/py3.12/windows.txt @@ -132,6 +132,8 @@ wheel==0.38.4 # via -r requirements/windows.txt wmi==1.5.1 # via -r requirements/windows.txt +xmltodict==0.13.0 + # via -r requirements/windows.txt zc.lockfile==2.0 # via cherrypy zipp==3.12.0 diff --git a/requirements/static/pkg/py3.7/windows.txt b/requirements/static/pkg/py3.7/windows.txt index 334db47befb..c9f733ae702 100644 --- a/requirements/static/pkg/py3.7/windows.txt +++ b/requirements/static/pkg/py3.7/windows.txt @@ -137,6 +137,8 @@ wheel==0.38.4 # via -r requirements/windows.txt wmi==1.5.1 # via -r requirements/windows.txt +xmltodict==0.13.0 + # via -r requirements/windows.txt zc.lockfile==2.0 # via cherrypy zipp==3.5.0 diff --git a/requirements/static/pkg/py3.8/windows.txt b/requirements/static/pkg/py3.8/windows.txt index f304db73f73..2f1c4e98f06 100644 --- a/requirements/static/pkg/py3.8/windows.txt +++ b/requirements/static/pkg/py3.8/windows.txt @@ -133,6 +133,8 @@ wheel==0.38.4 # via -r requirements/windows.txt wmi==1.5.1 # via -r requirements/windows.txt +xmltodict==0.13.0 + # via -r requirements/windows.txt zc.lockfile==2.0 # via cherrypy zipp==3.5.0 diff --git a/requirements/static/pkg/py3.9/windows.txt b/requirements/static/pkg/py3.9/windows.txt index 3ebb43358ef..626ebf5333d 100644 --- a/requirements/static/pkg/py3.9/windows.txt +++ b/requirements/static/pkg/py3.9/windows.txt @@ -133,6 +133,8 @@ wheel==0.38.4 # via -r requirements/windows.txt wmi==1.5.1 # via -r requirements/windows.txt +xmltodict==0.13.0 + # via -r requirements/windows.txt zc.lockfile==2.0 # via cherrypy zipp==3.5.0 diff --git a/requirements/windows.txt b/requirements/windows.txt index 19ae0c889df..05884f31a70 100644 --- a/requirements/windows.txt +++ b/requirements/windows.txt @@ -29,5 +29,6 @@ urllib3>=1.26.5 # # watchdog>=2.1.3 wheel>=0.38.1 +xmltodict>=0.13.0 importlib-metadata>=3.3.0 From 9baaab21351a1fcbc10ff3fb04cf6b98965b16ec Mon Sep 17 00:00:00 2001 From: Shane Lee Date: Tue, 23 Jan 2024 08:06:52 -0700 Subject: [PATCH 23/29] Add some import tests for the package --- .../pkg/integration/test_check_imports.py | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/pytests/pkg/integration/test_check_imports.py b/tests/pytests/pkg/integration/test_check_imports.py index f16aa338bf7..0397b105b00 100644 --- a/tests/pytests/pkg/integration/test_check_imports.py +++ b/tests/pytests/pkg/integration/test_check_imports.py @@ -15,6 +15,7 @@ log = logging.getLogger(__name__) CHECK_IMPORTS_SLS_CONTENTS = """ #!py import importlib +import sys def run(): config = {} @@ -81,7 +82,12 @@ def run(): ] } - for import_name in ["telnetlib"]: + # Import required for all OS'es + for import_name in [ + "jinja2", + "telnetlib", + "yaml", + ]: try: importlib.import_module(import_name) config[import_name] = { @@ -101,6 +107,40 @@ def run(): } ] } + + # Windows specific requirements (I think, there may be some for other OSes in here) + if sys.platform == "win32": + for import_name in [ + "cffi", + "clr_loader", + "lxml", + "pythonnet", + "pytz", + "pywintypes", + "timelib", + "win32", + "wmi", + "xmltodict", + ]: + try: + importlib.import_module(import_name) + config[import_name] = { + 'test.succeed_without_changes': [ + { + "name": import_name, + 'comment': "The '{}' import succeeded.".format(import_name) + } + ] + } + except ModuleNotFoundError as err: + config[import_name] = { + 'test.fail_without_changes': [ + { + "name": import_name, + 'comment': "The '{}' import failed. The error was: {}".format(import_name, err) + } + ] + } return config """ From 20fd342182c448fd3f21015b3aed0dba45380d09 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 23 Jan 2024 12:46:21 +0000 Subject: [PATCH 24/29] Only show system information. The test plan was always wrong due to the kind of test selection that was done. Signed-off-by: Pedro Algarvio --- .../test-package-downloads-action.yml.jinja | 46 +++---------------- .github/workflows/test-action-linux.yml | 4 +- .github/workflows/test-action-macos.yml | 11 ++--- .github/workflows/test-action-windows.yml | 4 +- .../test-package-downloads-action.yml | 46 +++---------------- .../workflows/test-packages-action-linux.yml | 7 ++- .../workflows/test-packages-action-macos.yml | 11 ++--- .../test-packages-action-windows.yml | 7 ++- noxfile.py | 6 +++ requirements/pytest.txt | 2 +- requirements/static/ci/py3.10/cloud.txt | 2 +- requirements/static/ci/py3.10/darwin.txt | 2 +- requirements/static/ci/py3.10/freebsd.txt | 2 +- requirements/static/ci/py3.10/linux.txt | 2 +- requirements/static/ci/py3.10/windows.txt | 2 +- requirements/static/ci/py3.11/cloud.txt | 2 +- requirements/static/ci/py3.11/darwin.txt | 2 +- requirements/static/ci/py3.11/freebsd.txt | 2 +- requirements/static/ci/py3.11/linux.txt | 2 +- requirements/static/ci/py3.11/windows.txt | 2 +- requirements/static/ci/py3.12/cloud.txt | 2 +- requirements/static/ci/py3.12/darwin.txt | 2 +- requirements/static/ci/py3.12/freebsd.txt | 2 +- requirements/static/ci/py3.12/linux.txt | 2 +- requirements/static/ci/py3.12/windows.txt | 2 +- requirements/static/ci/py3.7/cloud.txt | 2 +- requirements/static/ci/py3.7/freebsd.txt | 2 +- requirements/static/ci/py3.7/linux.txt | 2 +- requirements/static/ci/py3.7/windows.txt | 2 +- requirements/static/ci/py3.8/cloud.txt | 2 +- requirements/static/ci/py3.8/freebsd.txt | 2 +- requirements/static/ci/py3.8/linux.txt | 2 +- requirements/static/ci/py3.8/windows.txt | 2 +- requirements/static/ci/py3.9/cloud.txt | 2 +- requirements/static/ci/py3.9/darwin.txt | 2 +- requirements/static/ci/py3.9/freebsd.txt | 2 +- requirements/static/ci/py3.9/linux.txt | 2 +- requirements/static/ci/py3.9/windows.txt | 2 +- tools/vm.py | 21 +++++++++ 39 files changed, 84 insertions(+), 137 deletions(-) diff --git a/.github/workflows/templates/test-package-downloads-action.yml.jinja b/.github/workflows/templates/test-package-downloads-action.yml.jinja index 6fb293cf971..ee7f19c7175 100644 --- a/.github/workflows/templates/test-package-downloads-action.yml.jinja +++ b/.github/workflows/templates/test-package-downloads-action.yml.jinja @@ -129,22 +129,9 @@ jobs: run: | tools --timestamps vm decompress-dependencies ${{ matrix.distro-slug }} - - name: Show System Info & Test Plan - env: - SALT_RELEASE: "${{ inputs.salt-version }}" - SALT_REPO_ARCH: ${{ matrix.arch }} - SALT_REPO_TYPE: ${{ inputs.environment }} - SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }} - SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }} - SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }} - SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }} - SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" - LATEST_SALT_RELEASE: "${{ inputs.latest-release }}" - DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }} + - name: Show System Info run: | - tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \ - -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE -E DOWNLOAD_TEST_PACKAGE_TYPE \ + tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \ --nox-session=${{ inputs.nox-session }}-pkgs ${{ matrix.distro-slug }} -- download-pkgs - name: Run Package Download Tests @@ -341,18 +328,10 @@ jobs: run: | nox --force-color -e decompress-dependencies -- macos ${{ matrix.arch }} - - name: Show System Info & Test Plan + - name: Show System Info env: - SALT_RELEASE: "${{ inputs.salt-version }}" SKIP_REQUIREMENTS_INSTALL: "1" - PRINT_TEST_SELECTION: "1" - PRINT_TEST_PLAN_ONLY: "1" - PRINT_SYSTEM_INFO: "1" - GITHUB_ACTIONS_PIPELINE: "1" - SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" - SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" - LATEST_SALT_RELEASE: "${{ inputs.latest-release }}" - DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }} + PRINT_SYSTEM_INFO_ONLY: "1" run: | sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- download-pkgs @@ -556,22 +535,9 @@ jobs: run: | tools --timestamps vm decompress-dependencies ${{ matrix.distro-slug }} - - name: Show System Info & Test Plan - env: - SALT_RELEASE: "${{ inputs.salt-version }}" - SALT_REPO_ARCH: ${{ matrix.arch }} - LATEST_SALT_RELEASE: "${{ inputs.latest-release }}" - SALT_REPO_TYPE: ${{ inputs.environment }} - SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }} - SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }} - SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }} - SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }} - SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" - DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }} + - name: Show System Info run: | - tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \ - -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE -E DOWNLOAD_TEST_PACKAGE_TYPE \ + tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \ --nox-session=${{ inputs.nox-session }}-pkgs ${{ matrix.distro-slug }} -- download-pkgs - name: Run Package Download Tests diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index cca9cc029fe..287748e48db 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -194,9 +194,9 @@ jobs: run: | tools --timestamps vm decompress-dependencies ${{ inputs.distro-slug }} - - name: Show System Info & Test Plan + - name: Show System Info run: | - tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \ + tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \ --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }} \ ${{ matrix.tests-chunk }} diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 19cd30f2f3f..24465a724c0 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -166,17 +166,12 @@ jobs: with: name: testrun-changed-files.txt - - name: Show System Info & Test Plan + - name: Show System Info env: SKIP_REQUIREMENTS_INSTALL: "1" - PRINT_TEST_SELECTION: "1" - PRINT_TEST_PLAN_ONLY: "1" - PRINT_SYSTEM_INFO: "1" - GITHUB_ACTIONS_PIPELINE: "1" - SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" - SKIP_CODE_COVERAGE: "1" + PRINT_SYSTEM_INFO_ONLY: "1" run: | - sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- -k "mac or darwin" + sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} - name: Run Changed Tests id: run-fast-changed-tests diff --git a/.github/workflows/test-action-windows.yml b/.github/workflows/test-action-windows.yml index 4a7d496042c..92b048f570e 100644 --- a/.github/workflows/test-action-windows.yml +++ b/.github/workflows/test-action-windows.yml @@ -194,9 +194,9 @@ jobs: run: | tools --timestamps vm decompress-dependencies ${{ inputs.distro-slug }} - - name: Show System Info & Test Plan + - name: Show System Info run: | - tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \ + tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \ --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }} \ ${{ matrix.tests-chunk }} diff --git a/.github/workflows/test-package-downloads-action.yml b/.github/workflows/test-package-downloads-action.yml index 8f0f977a26c..a031416d943 100644 --- a/.github/workflows/test-package-downloads-action.yml +++ b/.github/workflows/test-package-downloads-action.yml @@ -274,22 +274,9 @@ jobs: run: | tools --timestamps vm decompress-dependencies ${{ matrix.distro-slug }} - - name: Show System Info & Test Plan - env: - SALT_RELEASE: "${{ inputs.salt-version }}" - SALT_REPO_ARCH: ${{ matrix.arch }} - SALT_REPO_TYPE: ${{ inputs.environment }} - SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }} - SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }} - SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }} - SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }} - SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" - LATEST_SALT_RELEASE: "${{ inputs.latest-release }}" - DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }} + - name: Show System Info run: | - tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \ - -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE -E DOWNLOAD_TEST_PACKAGE_TYPE \ + tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \ --nox-session=${{ inputs.nox-session }}-pkgs ${{ matrix.distro-slug }} -- download-pkgs - name: Run Package Download Tests @@ -493,18 +480,10 @@ jobs: run: | nox --force-color -e decompress-dependencies -- macos ${{ matrix.arch }} - - name: Show System Info & Test Plan + - name: Show System Info env: - SALT_RELEASE: "${{ inputs.salt-version }}" SKIP_REQUIREMENTS_INSTALL: "1" - PRINT_TEST_SELECTION: "1" - PRINT_TEST_PLAN_ONLY: "1" - PRINT_SYSTEM_INFO: "1" - GITHUB_ACTIONS_PIPELINE: "1" - SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" - SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" - LATEST_SALT_RELEASE: "${{ inputs.latest-release }}" - DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }} + PRINT_SYSTEM_INFO_ONLY: "1" run: | sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- download-pkgs @@ -712,22 +691,9 @@ jobs: run: | tools --timestamps vm decompress-dependencies ${{ matrix.distro-slug }} - - name: Show System Info & Test Plan - env: - SALT_RELEASE: "${{ inputs.salt-version }}" - SALT_REPO_ARCH: ${{ matrix.arch }} - LATEST_SALT_RELEASE: "${{ inputs.latest-release }}" - SALT_REPO_TYPE: ${{ inputs.environment }} - SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }} - SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }} - SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }} - SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }} - SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" - DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }} + - name: Show System Info run: | - tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \ - -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE -E DOWNLOAD_TEST_PACKAGE_TYPE \ + tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \ --nox-session=${{ inputs.nox-session }}-pkgs ${{ matrix.distro-slug }} -- download-pkgs - name: Run Package Download Tests diff --git a/.github/workflows/test-packages-action-linux.yml b/.github/workflows/test-packages-action-linux.yml index db8fe9b5efb..ae85b3989ac 100644 --- a/.github/workflows/test-packages-action-linux.yml +++ b/.github/workflows/test-packages-action-linux.yml @@ -190,11 +190,10 @@ jobs: # This step can go away once we stop testing classic packages upgrade/downgrades to/from 3005.x tools --timestamps vm ssh ${{ inputs.distro-slug }} -- "sudo python3 -m pip install -U 'importlib-metadata<=4.13.0' 'virtualenv<=20.21.1'" - - name: Show System Info & Test Plan + - name: Show System Info run: | - tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \ - --nox-session=${{ inputs.nox-session }}-pkgs ${{ inputs.distro-slug }} -- ${{ matrix.tests-chunk }} \ - ${{ matrix.version && format('--prev-version {0}', matrix.version) || ''}} + tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \ + --nox-session=${{ inputs.nox-session }}-pkgs ${{ inputs.distro-slug }} -- ${{ matrix.tests-chunk }} - name: Run Package Tests run: | diff --git a/.github/workflows/test-packages-action-macos.yml b/.github/workflows/test-packages-action-macos.yml index 0db11075663..2b6b2e9ca90 100644 --- a/.github/workflows/test-packages-action-macos.yml +++ b/.github/workflows/test-packages-action-macos.yml @@ -162,17 +162,12 @@ jobs: run: | nox --force-color -e decompress-dependencies -- macos ${{ inputs.arch }} - - name: Show System Info & Test Plan + - name: Show System Info env: SKIP_REQUIREMENTS_INSTALL: "1" - PRINT_TEST_SELECTION: "1" - PRINT_TEST_PLAN_ONLY: "1" - PRINT_SYSTEM_INFO: "1" - GITHUB_ACTIONS_PIPELINE: "1" - SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + PRINT_SYSTEM_INFO_ONLY: "1" run: | - sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- ${{ matrix.tests-chunk }} \ - ${{ matrix.version && format('--prev-version {0}', matrix.version) || ''}} + sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- ${{ matrix.tests-chunk }} - name: Run Package Tests env: diff --git a/.github/workflows/test-packages-action-windows.yml b/.github/workflows/test-packages-action-windows.yml index bca0b852592..1bf81b75824 100644 --- a/.github/workflows/test-packages-action-windows.yml +++ b/.github/workflows/test-packages-action-windows.yml @@ -190,11 +190,10 @@ jobs: # This step can go away once we stop testing classic packages upgrade/downgrades to/from 3005.x tools --timestamps vm ssh ${{ inputs.distro-slug }} -- "sudo python3 -m pip install -U 'importlib-metadata<=4.13.0' 'virtualenv<=20.21.1'" - - name: Show System Info & Test Plan + - name: Show System Info run: | - tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \ - --nox-session=${{ inputs.nox-session }}-pkgs ${{ inputs.distro-slug }} -- ${{ matrix.tests-chunk }} \ - ${{ matrix.version && format('--prev-version {0}', matrix.version) || ''}} + tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \ + --nox-session=${{ inputs.nox-session }}-pkgs ${{ inputs.distro-slug }} -- ${{ matrix.tests-chunk }} - name: Run Package Tests run: | diff --git a/noxfile.py b/noxfile.py index 003afbdbe87..2ed8e4d0a00 100644 --- a/noxfile.py +++ b/noxfile.py @@ -62,6 +62,7 @@ if PRINT_SYSTEM_INFO is None: PRINT_SYSTEM_INFO = CI_RUN else: PRINT_SYSTEM_INFO = PRINT_SYSTEM_INFO == "1" +PRINT_SYSTEM_INFO_ONLY = os.environ.get("PRINT_SYSTEM_INFO_ONLY", "0") == "1" SKIP_REQUIREMENTS_INSTALL = os.environ.get("SKIP_REQUIREMENTS_INSTALL", "0") == "1" EXTRA_REQUIREMENTS_INSTALL = os.environ.get("EXTRA_REQUIREMENTS_INSTALL") COVERAGE_REQUIREMENT = os.environ.get("COVERAGE_REQUIREMENT") @@ -1015,6 +1016,11 @@ def _pytest(session, coverage, cmd_args, env=None, on_rerun=False): args.append("--log-file={}".format(RUNTESTS_LOGFILE)) args.extend(cmd_args) + if PRINT_SYSTEM_INFO_ONLY and "--sys-info-and-exit" not in args: + args.append("--sys-info-and-exit") + session.run("python", "-m", "pytest", *args, env=env) + return + if PRINT_SYSTEM_INFO and "--sysinfo" not in args: args.append("--sysinfo") diff --git a/requirements/pytest.txt b/requirements/pytest.txt index 23e83b8092c..950867df605 100644 --- a/requirements/pytest.txt +++ b/requirements/pytest.txt @@ -2,7 +2,7 @@ mock >= 3.0.0 # PyTest docker pytest >= 7.2.0 -pytest-salt-factories >= 1.0.0rc28 +pytest-salt-factories >= 1.0.0rc29 pytest-helpers-namespace >= 2019.1.8 pytest-subtests pytest-timeout diff --git a/requirements/static/ci/py3.10/cloud.txt b/requirements/static/ci/py3.10/cloud.txt index b891ab132a1..2abef7dc72f 100644 --- a/requirements/static/ci/py3.10/cloud.txt +++ b/requirements/static/ci/py3.10/cloud.txt @@ -471,7 +471,7 @@ pytest-httpserver==1.0.8 # via # -c requirements/static/ci/py3.10/linux.txt # -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via # -c requirements/static/ci/py3.10/linux.txt # -r requirements/pytest.txt diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index 632dd4d7391..49ff4eda395 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -331,7 +331,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index 198053f59da..4dad13a2b3d 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -324,7 +324,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index b017f0a3ee1..ad1636529a6 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -341,7 +341,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.10/windows.txt b/requirements/static/ci/py3.10/windows.txt index 05dcf33f303..1da9f4396f5 100644 --- a/requirements/static/ci/py3.10/windows.txt +++ b/requirements/static/ci/py3.10/windows.txt @@ -309,7 +309,7 @@ pytest-helpers-namespace==2021.12.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.11/cloud.txt b/requirements/static/ci/py3.11/cloud.txt index d2d4afcc69d..d15bdd12fac 100644 --- a/requirements/static/ci/py3.11/cloud.txt +++ b/requirements/static/ci/py3.11/cloud.txt @@ -435,7 +435,7 @@ pytest-httpserver==1.0.8 # via # -c requirements/static/ci/py3.11/linux.txt # -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via # -c requirements/static/ci/py3.11/linux.txt # -r requirements/pytest.txt diff --git a/requirements/static/ci/py3.11/darwin.txt b/requirements/static/ci/py3.11/darwin.txt index c8f787d31e5..afddd1d39aa 100644 --- a/requirements/static/ci/py3.11/darwin.txt +++ b/requirements/static/ci/py3.11/darwin.txt @@ -302,7 +302,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.11/freebsd.txt b/requirements/static/ci/py3.11/freebsd.txt index 34cd70a2d48..e02c2cc4c32 100644 --- a/requirements/static/ci/py3.11/freebsd.txt +++ b/requirements/static/ci/py3.11/freebsd.txt @@ -301,7 +301,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.11/linux.txt b/requirements/static/ci/py3.11/linux.txt index 0869197b905..c8fb1c4e96c 100644 --- a/requirements/static/ci/py3.11/linux.txt +++ b/requirements/static/ci/py3.11/linux.txt @@ -318,7 +318,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.11/windows.txt b/requirements/static/ci/py3.11/windows.txt index 358ff340067..06672cb1455 100644 --- a/requirements/static/ci/py3.11/windows.txt +++ b/requirements/static/ci/py3.11/windows.txt @@ -305,7 +305,7 @@ pytest-helpers-namespace==2021.12.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.12/cloud.txt b/requirements/static/ci/py3.12/cloud.txt index 7d05aded66a..b00e9596234 100644 --- a/requirements/static/ci/py3.12/cloud.txt +++ b/requirements/static/ci/py3.12/cloud.txt @@ -435,7 +435,7 @@ pytest-httpserver==1.0.8 # via # -c requirements/static/ci/py3.12/linux.txt # -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via # -c requirements/static/ci/py3.12/linux.txt # -r requirements/pytest.txt diff --git a/requirements/static/ci/py3.12/darwin.txt b/requirements/static/ci/py3.12/darwin.txt index 770ac1d7902..9160cc7e38c 100644 --- a/requirements/static/ci/py3.12/darwin.txt +++ b/requirements/static/ci/py3.12/darwin.txt @@ -302,7 +302,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.12/freebsd.txt b/requirements/static/ci/py3.12/freebsd.txt index 9c3e9e51ddb..ebd1645bd69 100644 --- a/requirements/static/ci/py3.12/freebsd.txt +++ b/requirements/static/ci/py3.12/freebsd.txt @@ -301,7 +301,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.12/linux.txt b/requirements/static/ci/py3.12/linux.txt index 7c64e8eac5b..7f98b8c08cb 100644 --- a/requirements/static/ci/py3.12/linux.txt +++ b/requirements/static/ci/py3.12/linux.txt @@ -318,7 +318,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.12/windows.txt b/requirements/static/ci/py3.12/windows.txt index d943b70255b..ae67b7afa56 100644 --- a/requirements/static/ci/py3.12/windows.txt +++ b/requirements/static/ci/py3.12/windows.txt @@ -305,7 +305,7 @@ pytest-helpers-namespace==2021.12.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.7/cloud.txt b/requirements/static/ci/py3.7/cloud.txt index 81efd207812..fbe29c8c3e6 100644 --- a/requirements/static/ci/py3.7/cloud.txt +++ b/requirements/static/ci/py3.7/cloud.txt @@ -522,7 +522,7 @@ pytest-httpserver==1.0.6 # via # -c requirements/static/ci/py3.7/linux.txt # -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via # -c requirements/static/ci/py3.7/linux.txt # -r requirements/pytest.txt diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt index f3171516c07..88a3d1914a7 100644 --- a/requirements/static/ci/py3.7/freebsd.txt +++ b/requirements/static/ci/py3.7/freebsd.txt @@ -366,7 +366,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.6 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.7/linux.txt b/requirements/static/ci/py3.7/linux.txt index 36708ecfa88..5b17d49164e 100644 --- a/requirements/static/ci/py3.7/linux.txt +++ b/requirements/static/ci/py3.7/linux.txt @@ -378,7 +378,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.6 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.7/windows.txt b/requirements/static/ci/py3.7/windows.txt index c942618b2ce..dba21dfd663 100644 --- a/requirements/static/ci/py3.7/windows.txt +++ b/requirements/static/ci/py3.7/windows.txt @@ -323,7 +323,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.6 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.8/cloud.txt b/requirements/static/ci/py3.8/cloud.txt index 977351155da..48208fbcded 100644 --- a/requirements/static/ci/py3.8/cloud.txt +++ b/requirements/static/ci/py3.8/cloud.txt @@ -509,7 +509,7 @@ pytest-httpserver==1.0.8 # via # -c requirements/static/ci/py3.8/linux.txt # -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via # -c requirements/static/ci/py3.8/linux.txt # -r requirements/pytest.txt diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index 777b0b4eb7c..0a2ed4459d2 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -353,7 +353,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.8/linux.txt b/requirements/static/ci/py3.8/linux.txt index a71925f5074..bd2983159f5 100644 --- a/requirements/static/ci/py3.8/linux.txt +++ b/requirements/static/ci/py3.8/linux.txt @@ -365,7 +365,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.8/windows.txt b/requirements/static/ci/py3.8/windows.txt index 02ae00bd118..c724b70e723 100644 --- a/requirements/static/ci/py3.8/windows.txt +++ b/requirements/static/ci/py3.8/windows.txt @@ -310,7 +310,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.9/cloud.txt b/requirements/static/ci/py3.9/cloud.txt index 6d2b7726781..0a7d77d0f79 100644 --- a/requirements/static/ci/py3.9/cloud.txt +++ b/requirements/static/ci/py3.9/cloud.txt @@ -511,7 +511,7 @@ pytest-httpserver==1.0.8 # via # -c requirements/static/ci/py3.9/linux.txt # -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via # -c requirements/static/ci/py3.9/linux.txt # -r requirements/pytest.txt diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index 6c8e1fa6327..497a510321c 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -362,7 +362,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index a05648451a1..59ed8b4c4cf 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -355,7 +355,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index 911b3457437..b47b8d5ce5c 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -365,7 +365,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/requirements/static/ci/py3.9/windows.txt b/requirements/static/ci/py3.9/windows.txt index 4c1e6281536..97c87cadde5 100644 --- a/requirements/static/ci/py3.9/windows.txt +++ b/requirements/static/ci/py3.9/windows.txt @@ -311,7 +311,7 @@ pytest-helpers-namespace==2021.4.29 # pytest-shell-utilities pytest-httpserver==1.0.8 # via -r requirements/pytest.txt -pytest-salt-factories==1.0.0rc28 +pytest-salt-factories==1.0.0rc29 # via -r requirements/pytest.txt pytest-shell-utilities==1.8.0 # via pytest-salt-factories diff --git a/tools/vm.py b/tools/vm.py index 6a65c189f63..996ebc845d1 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -263,6 +263,22 @@ def rsync(ctx: Context, name: str, download: bool = False): "--print-tests-selection", ], }, + "print_system_info": { + "help": "Print the system information", + "action": "store_true", + "flags": [ + "--psi", + "--print-system-information", + ], + }, + "print_system_info_only": { + "help": "Print the system information and exit", + "action": "store_true", + "flags": [ + "--psio", + "--print-system-information-only", + ], + }, "skip_code_coverage": { "help": "Skip tracking code coverage", "action": "store_true", @@ -293,6 +309,7 @@ def test( skip_requirements_install: bool = False, print_tests_selection: bool = False, print_system_info: bool = False, + print_system_info_only: bool = False, skip_code_coverage: bool = False, envvars: list[str] = None, fips: bool = False, @@ -323,6 +340,10 @@ def test( env["PRINT_SYSTEM_INFO"] = "1" else: env["PRINT_SYSTEM_INFO"] = "0" + if print_system_info_only: + env["PRINT_SYSTEM_INFO_ONLY"] = "1" + else: + env["PRINT_SYSTEM_INFO_ONLY"] = "0" if ( skip_requirements_install or os.environ.get("SKIP_REQUIREMENTS_INSTALL", "0") == "1" From 121301514d20b6a9f3926198919160913fd0a513 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 24 Jan 2024 15:15:33 +0000 Subject: [PATCH 25/29] Minor cleanup Signed-off-by: Pedro Algarvio --- tests/pytests/functional/utils/test_http.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/tests/pytests/functional/utils/test_http.py b/tests/pytests/functional/utils/test_http.py index f45b4409b31..646d2f1c5fb 100644 --- a/tests/pytests/functional/utils/test_http.py +++ b/tests/pytests/functional/utils/test_http.py @@ -3,17 +3,16 @@ import ssl import tarfile import pytest - -try: - import trustme -except ImportError: - pass - from pytestshellutils.utils import ports from saltfactories.utils import random_string import salt.utils.http +pytestmark = [ + pytest.mark.slow_test, + pytest.mark.skip_if_binaries_missing("docker", "dockerd", check_all=False), +] + @pytest.mark.parametrize("backend", ["requests", "urllib2", "tornado"]) def test_decode_body(webserver, integration_files_dir, backend): @@ -26,12 +25,6 @@ def test_decode_body(webserver, integration_files_dir, backend): assert isinstance(ret["body"], bytes) -pytestmark = [ - pytest.mark.slow_test, - pytest.mark.skip_if_binaries_missing("docker", "dockerd", check_all=False), -] - - @pytest.fixture(scope="module") def tinyproxy_port(): return ports.get_unused_localhost_port() @@ -49,6 +42,7 @@ def tinyproxy_pass(): @pytest.fixture(scope="session") def ca(): + trustme = pytest.importorskip("trustme") return trustme.CA() From 560b24f1a65fc5219ccc8c587f0d7dff03bb5f12 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 25 Jan 2024 16:13:26 +0000 Subject: [PATCH 26/29] Fix a few `yield` statements which should have been converted to `await` Signed-off-by: Pedro Algarvio --- salt/transport/zeromq.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index 1caed8a23b3..73db20586ea 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -968,18 +968,18 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer): htopic = salt.utils.stringutils.to_bytes( hashlib.sha1(salt.utils.stringutils.to_bytes(topic)).hexdigest() ) - yield self.dpub_sock.send_multipart([htopic, payload]) + await self.dpub_sock.send_multipart([htopic, payload]) log.trace("Filtered data has been sent") # Syndic broadcast if self.opts.get("order_masters"): log.trace("Sending filtered data to syndic") - yield self.dpub_sock.send_multipart([b"syndic", payload]) + await self.dpub_sock.send_multipart([b"syndic", payload]) log.trace("Filtered data has been sent to syndic") # otherwise its a broadcast else: # TODO: constants file for "broadcast" log.trace("Sending broadcasted data over publisher %s", self.pub_uri) - yield self.dpub_sock.send_multipart([b"broadcast", payload]) + await self.dpub_sock.send_multipart([b"broadcast", payload]) log.trace("Broadcasted data has been sent") else: log.trace("Sending ZMQ-unfiltered data over publisher %s", self.pub_uri) From 3afeaad0a7fe976c1872e4ef2a038e2f8ca5bf2b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 25 Jan 2024 18:03:19 +0000 Subject: [PATCH 27/29] Try to create the directory and don't fail if it already exists Signed-off-by: Pedro Algarvio --- salt/minion.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index addc91fbc86..ad23f46e1f5 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -290,9 +290,8 @@ def get_proc_dir(cachedir, **kwargs): else: mode = {"mode": mode} - if not os.path.isdir(fn_): - # proc_dir is not present, create it with mode settings - os.makedirs(fn_, **mode) + # proc_dir is not present, create it with mode settings + os.makedirs(fn_, **mode, exist_ok=True) d_stat = os.stat(fn_) @@ -952,8 +951,7 @@ class SMinion(MinionBase): import salt.utils.yaml pdir = os.path.join(self.opts["cachedir"], "pillar") - if not os.path.isdir(pdir): - os.makedirs(pdir, 0o700) + os.makedirs(pdir, 0o700, exist_ok=True) ptop = os.path.join(pdir, "top.sls") if self.opts["saltenv"] is not None: penv = self.opts["saltenv"] From 890df5021bf4ed20009b12286d04b3e4fb9ec925 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 26 Jan 2024 05:54:45 +0000 Subject: [PATCH 28/29] Exit the process cleanly Signed-off-by: Pedro Algarvio --- tests/support/pytest/transport.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/support/pytest/transport.py b/tests/support/pytest/transport.py index e63a330cf7c..bfb69baa3ad 100644 --- a/tests/support/pytest/transport.py +++ b/tests/support/pytest/transport.py @@ -281,13 +281,14 @@ class PubServerChannelProcess(salt.utils.process.SignalHandlingProcess): ) def run(self): - ioloop = tornado.ioloop.IOLoop() try: while True: try: payload = self.queue.get(False) except queue.Empty: + if self._closing is True: + break time.sleep(0.03) continue if payload is None: From 2631170a6f0a2051b0c709a49ea98b00caf2f4b3 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 26 Jan 2024 06:20:11 +0000 Subject: [PATCH 29/29] Fix tests not properly fixed during the merge-forward Signed-off-by: Pedro Algarvio --- .../zeromq/test_pub_server_channel.py | 11 ++++++++-- tests/support/pytest/transport.py | 21 +++++++++---------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py index 790d3ce2bde..2fc630d6efc 100644 --- a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py +++ b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py @@ -7,6 +7,7 @@ import time from contextlib import contextmanager import pytest +import tornado.ioloop from saltfactories.utils import random_string import salt.transport.zeromq @@ -32,9 +33,15 @@ class PubServerChannelSender: self.payload_list = payload_list def run(self): + loop = tornado.ioloop.IOLoop() + loop.add_callback(self._run, loop) + loop.start() + + async def _run(self, loop): for payload in self.payload_list: - self.pub_server_channel.publish(payload) - time.sleep(2) + await self.pub_server_channel.publish(payload) + await asyncio.sleep(2) + loop.stop() def generate_msg_list(msg_cnt, minions_list, broadcast): diff --git a/tests/support/pytest/transport.py b/tests/support/pytest/transport.py index bfb69baa3ad..2b4e62cc24d 100644 --- a/tests/support/pytest/transport.py +++ b/tests/support/pytest/transport.py @@ -7,7 +7,6 @@ import socket import time import pytest -import tornado.gen import tornado.ioloop import tornado.iostream import zmq @@ -118,8 +117,7 @@ class Collector(salt.utils.process.SignalHandlingProcess): break self.sock = tornado.iostream.IOStream(sock) - @tornado.gen.coroutine - def _recv(self): + async def _recv(self): if self.transport == "zeromq": # test_zeromq_filtering requires catching the # SaltDeserializationError in order to pass. @@ -138,26 +136,25 @@ class Collector(salt.utils.process.SignalHandlingProcess): log.debug( "Publish received for not this minion: %s", message_target ) - raise salt.ext.tornado.gen.Return(None) + return serial_payload = salt.payload.loads(messages[1]) else: raise Exception("Invalid number of messages") - raise salt.ext.tornado.gen.Return(serial_payload) + return serial_payload except (zmq.ZMQError, salt.exceptions.SaltDeserializationError): raise RecvError("ZMQ Error") else: for msg in self.unpacker: serial_payload = salt.payload.loads(msg["body"]) - raise tornado.gen.Return(serial_payload) - byts = yield self.sock.read_bytes(8096, partial=True) + return serial_payload + byts = await self.sock.read_bytes(8096, partial=True) self.unpacker.feed(byts) for msg in self.unpacker: serial_payload = salt.payload.loads(msg["body"]) - raise tornado.gen.Return(serial_payload) + return serial_payload raise RecvError("TCP Error") - @tornado.gen.coroutine - def _run(self, loop): + async def _run(self, loop): try: self._setup_listener() except Exception: # pylint: disable=broad-except @@ -179,7 +176,7 @@ class Collector(salt.utils.process.SignalHandlingProcess): log.error("Receive timeout reached in test collector!") break try: - payload = yield self._recv() + payload = await self._recv() except RecvError: time.sleep(0.03) else: @@ -329,6 +326,7 @@ class PubServerChannelProcess(salt.utils.process.SignalHandlingProcess): self.publish({"tgt_type": "glob", "tgt": "*", "jid": -1, "start": True}) if self.collector.running.wait(1) is True: break + time.sleep(0.5) attempts -= 1 else: pytest.fail("Failed to confirm the collector has started") @@ -341,6 +339,7 @@ class PubServerChannelProcess(salt.utils.process.SignalHandlingProcess): self.publish({"tgt_type": "glob", "tgt": "*", "jid": -1, "stop": True}) if self.collector.stop_running.wait(1) is True: break + time.sleep(0.5) attempts -= 1 else: pytest.fail("Failed to confirm the collector has stopped")