From 26c550003f531250479d4dda263e6e792bff97ff Mon Sep 17 00:00:00 2001 From: nicholasmhughes Date: Tue, 13 Jun 2023 15:21:36 -0400 Subject: [PATCH 01/53] fixes saltstack/salt#64477 file.symlink will not replace/update existing symlink --- changelog/64477.fixed.md | 1 + salt/states/file.py | 4 +++- tests/pytests/functional/states/test_file.py | 22 ++++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 changelog/64477.fixed.md diff --git a/changelog/64477.fixed.md b/changelog/64477.fixed.md new file mode 100644 index 00000000000..d43f01714d9 --- /dev/null +++ b/changelog/64477.fixed.md @@ -0,0 +1 @@ +Fix file.symlink will not replace/update existing symlink diff --git a/salt/states/file.py b/salt/states/file.py index b40f5ef84c0..e00c07b57f1 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -1799,9 +1799,11 @@ def symlink( if __salt__["file.is_link"](name): # The link exists, verify that it matches the target - if os.path.normpath(__salt__["file.readlink"](name)) == os.path.normpath( + if os.path.normpath(__salt__["file.readlink"](name)) != os.path.normpath( target ): + __salt__["file.remove"](name) + else: if _check_symlink_ownership(name, user, group, win_owner): # The link looks good! if salt.utils.platform.is_windows(): diff --git a/tests/pytests/functional/states/test_file.py b/tests/pytests/functional/states/test_file.py index cea0552f365..a28ab64a35f 100644 --- a/tests/pytests/functional/states/test_file.py +++ b/tests/pytests/functional/states/test_file.py @@ -201,3 +201,25 @@ def test_file_managed_web_source_etag_operation( # The modified time of the cached file now changes assert cached_file_mtime != os.path.getmtime(cached_file) + + +def test_file_symlink_replace_existing_link(states, tmp_path): + # symlink name and target for state + name = tmp_path / "foo" + target = tmp_path / "baz" + + # create existing symlink to replace + old_target = tmp_path / "bar" + name.symlink_to(old_target) + + ret = states.file.symlink( + name=str(name), + target=str(target), + ) + + assert ret.filtered == { + "name": str(name), + "changes": {"new": str(name)}, + "comment": f"Created new symlink {str(name)} -> {str(target)}", + "result": True, + } From 367c4fda9c8edc811873f6b75ae04abcd8eb3b71 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 19 Jun 2023 14:06:31 +0100 Subject: [PATCH 02/53] Improved slack notifications Signed-off-by: Pedro Algarvio --- .github/workflows/nightly.yml | 52 ++++++- .github/workflows/templates/nightly.yml.jinja | 130 +++++++++++++++++- 2 files changed, 168 insertions(+), 14 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 75aea9f8afb..a5371f7878b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2103,18 +2103,56 @@ jobs: - name: Notify Slack id: slack + if: always() uses: slackapi/slack-github-action@v1.24.0 with: payload: | { - "text": "Nightly Workflow build result for the `${{ github.ref_name }}` branch(attempt: ${{ github.run_attempt }}): `${{ steps.get-workflow-info.outputs.conclusion }}`\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", - "blocks": [ + "attachments": [ { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Nightly Workflow build result for the `${{ github.ref_name }}` branch(attempt: ${{ github.run_attempt }}): `${{ steps.get-workflow-info.outputs.conclusion }}`\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - } + "fallback": "${{ github.workflow }} Workflow build result for the `${{ github.ref_name }}` branch(attempt: ${{ github.run_attempt }}): `${{ steps.get-workflow-info.outputs.conclusion }}`\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", + "color": "${{ steps.get-workflow-info.outputs.conclusion != 'success' && 'ff3d00' || '00e676' }}", + "fields": [ + { + "title": "Workflow", + "short": true, + "value": "${{ github.workflow }}", + "type": "mrkdwn" + }, + { + "title": "Workflow Run", + "short": true, + "value": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>", + "type": "mrkdwn" + }, + { + "title": "Branch", + "short": true, + "value": "${{ github.ref_name }}", + "type": "mrkdwn" + }, + { + "title": "Commit", + "short": true, + "value": "<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>", + "type": "mrkdwn" + }, + { + "title": "Attempt", + "short": true, + "value": "${{ github.run_attempt }}", + "type": "mrkdwn" + }, + { + "title": "Status", + "short": true, + "value": "${{ steps.get-workflow-info.outputs.conclusion }}", + "type": "mrkdwn" + } + ], + "author_name": "${{ github.event.sender.login }}", + "author_link": "${{ github.event.sender.html_url }}", + "author_icon": "${{ github.event.sender.avatar_url }}" } ] } diff --git a/.github/workflows/templates/nightly.yml.jinja b/.github/workflows/templates/nightly.yml.jinja index ae77bdda713..d36d0a72aa8 100644 --- a/.github/workflows/templates/nightly.yml.jinja +++ b/.github/workflows/templates/nightly.yml.jinja @@ -52,6 +52,81 @@ concurrency: <%- include "workflow-requirements-check.yml.jinja" %> <%- include "trigger-branch-workflows.yml.jinja" %> + {#- When we start using a slack app, we can update messages, not while using incoming webhooks + <%- if workflow_slug == "nightly" %> + + <%- do conclusion_needs.append('notify-slack') %> + notify-slack: + name: Notify Slack + runs-on: ubuntu-latest + environment: <{ gh_environment }> + needs: + <%- for need in prepare_workflow_needs.iter(consume=False) %> + - <{ need }> + <%- endfor %> + outputs: + update-ts: ${{ steps.slack.outputs.update-ts }} + steps: + - name: Notify Slack + id: slack + uses: slackapi/slack-github-action@v1.24.0 + with: + payload: | + { + "attachments": [ + { + "color": "ffca28", + "fields": [ + { + "title": "Workflow", + "short": true, + "value": "${{ github.workflow }}", + "type": "mrkdwn" + }, + { + "title": "Workflow Run", + "short": true, + "value": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>", + "type": "mrkdwn" + }, + { + "title": "Branch", + "short": true, + "value": "${{ github.ref_name }}", + "type": "mrkdwn" + }, + { + "title": "Commit", + "short": true, + "value": "<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>", + "type": "mrkdwn" + }, + { + "title": "Attempt", + "short": true, + "value": "${{ github.run_attempt }}", + "type": "mrkdwn" + }, + { + "title": "Status", + "short": true, + "value": "running", + "type": "mrkdwn" + } + ], + "author_name": "${{ github.event.sender.login }}", + "author_link": "${{ github.event.sender.html_url }}", + "author_icon": "${{ github.event.sender.avatar_url }}" + } + ] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + + <%- endif %> + #} + <%- endblock pre_jobs %> <%- block jobs %> @@ -123,18 +198,59 @@ concurrency: - name: Notify Slack id: slack + if: always() uses: slackapi/slack-github-action@v1.24.0 with: + {#- When we start using a slack app, we can update messages, not while using incoming webhooks + update-ts: ${{ needs.notify-slack.outputs.update-ts }} + #} payload: | { - "text": "Nightly Workflow build result for the `${{ github.ref_name }}` branch(attempt: ${{ github.run_attempt }}): `${{ steps.get-workflow-info.outputs.conclusion }}`\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", - "blocks": [ + "attachments": [ { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Nightly Workflow build result for the `${{ github.ref_name }}` branch(attempt: ${{ github.run_attempt }}): `${{ steps.get-workflow-info.outputs.conclusion }}`\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - } + "fallback": "${{ github.workflow }} Workflow build result for the `${{ github.ref_name }}` branch(attempt: ${{ github.run_attempt }}): `${{ steps.get-workflow-info.outputs.conclusion }}`\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", + "color": "${{ steps.get-workflow-info.outputs.conclusion != 'success' && 'ff3d00' || '00e676' }}", + "fields": [ + { + "title": "Workflow", + "short": true, + "value": "${{ github.workflow }}", + "type": "mrkdwn" + }, + { + "title": "Workflow Run", + "short": true, + "value": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>", + "type": "mrkdwn" + }, + { + "title": "Branch", + "short": true, + "value": "${{ github.ref_name }}", + "type": "mrkdwn" + }, + { + "title": "Commit", + "short": true, + "value": "<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>", + "type": "mrkdwn" + }, + { + "title": "Attempt", + "short": true, + "value": "${{ github.run_attempt }}", + "type": "mrkdwn" + }, + { + "title": "Status", + "short": true, + "value": "${{ steps.get-workflow-info.outputs.conclusion }}", + "type": "mrkdwn" + } + ], + "author_name": "${{ github.event.sender.login }}", + "author_link": "${{ github.event.sender.html_url }}", + "author_icon": "${{ github.event.sender.avatar_url }}" } ] } From 97e6ff7a1a2ae0e42a07ee1e6d1c8d024af23f95 Mon Sep 17 00:00:00 2001 From: Twangboy Date: Thu, 4 May 2023 18:28:43 -0600 Subject: [PATCH 03/53] Add scripts for setting up multi-minion on Windows --- pkg/windows/multi-master.cmd | 3 + pkg/windows/multi-minion.ps1 | 359 +++++++++++++++++++++++++++++++++++ 2 files changed, 362 insertions(+) create mode 100644 pkg/windows/multi-master.cmd create mode 100644 pkg/windows/multi-minion.ps1 diff --git a/pkg/windows/multi-master.cmd b/pkg/windows/multi-master.cmd new file mode 100644 index 00000000000..aa8dbc72e45 --- /dev/null +++ b/pkg/windows/multi-master.cmd @@ -0,0 +1,3 @@ +@ echo off +Set "CurDir=%~dp0" +PowerShell -ExecutionPolicy RemoteSigned -File "%CurDir%\multi-master.ps1" %* diff --git a/pkg/windows/multi-minion.ps1 b/pkg/windows/multi-minion.ps1 new file mode 100644 index 00000000000..855a8fef2d5 --- /dev/null +++ b/pkg/windows/multi-minion.ps1 @@ -0,0 +1,359 @@ +<# +.SYNOPSIS +Script for setting up an additional salt-minion on a machine with Salt installed + +.DESCRIPTION +This script will install an additional minion on a machine that already has a +Salt installtion using one of the Salt packages. It will set up the directory +structure required by Salt. It will also lay down a minion config to be used +by the Salt minion. Additionaly, this script will install and start a Salt +minion service that uses the root_dir and minion config. You can also pass the +name of a service account to be used by the service. + +You can also remove the multiminion setup with this script. + +This script should be run with Administrator privileges + +The following example will install a service named `salt-minion-mm10` that +starts with the LOCALSYSTEM account. It is the `-s` parameter that creates the +service: + +.EXAMPLE +PS>multi-minion.ps1 -Name mm10 -s + +The following example will install a service that starts with a user named +mmuser: + +.EXAMPLE +PS>multi-minion.ps1 -Name mm10 -s -m 192.168.0.10 -u mmuser -p secretword + +The following example will set up config for minion that can be run in the +background under a user account. Notice the command does not have the `-s` +parameter: + +.EXAMPLE +PS>multi-minion.ps1 -Name mm10 -m 192.168.0.10 + +The following example will remove a multiminion that has been installed with +this script: + +.EXAMPLE +PS>multi-minion.ps1 -Name mm10 -d + +#> + +[CmdletBinding()] +param( + + [Parameter(Mandatory=$true)] + [Alias("n")] + # The name used to create the service and root_dir. This is the only + # required parameter + [String] $Name, + + [Parameter(Mandatory=$false)] + [Alias("m")] + # The master to connect to. This can be an ip address or an fqdn. Default + # is salt + [String] $Master = "salt", + + [Parameter(Mandatory=$false)] + [Alias("r")] + # The root dir to place the minion config and directory structure. The + # default is %PROGRAMDATA%\Salt Project\Salt-$Name + [String] $RootDir = "$env:ProgramData\Salt Project\Salt-$Name", + + [Parameter(Mandatory=$false)] + [Alias("u")] + # User account to run the service under. The user account must be present on + # the system. The default is to use the LOCALSYSTEM account + [String] $User, + + [Parameter(Mandatory=$false)] + [Alias("p")] + # The password to the user account. Required if User is passed. We should + # probably figure out how to make this more secure + [String] $Password, + + [Parameter(Mandatory=$false)] + [Alias("s")] + # Set this switch to install the service. Default is to not install the + # service + [Switch] $Service, + + [Parameter(Mandatory=$false)] + [Alias("d")] + # Remove the specified multi-minion. All other parameters are ignored + [Switch] $Remove +) + +########################### Script Variables ############################# +$ssm_bin = "$env:ProgramFiles\Salt Project\Salt\ssm.exe" +$salt_bin = "$env:ProgramFiles\Salt Project\Salt\salt-minion.exe" +$service_name = "salt-minion-$($Name.ToLower())" +$default_root_dir = Resolve-Path -Path "$env:ProgramData\Salt Project\Salt" +$cache_dir = "$RootDir\var\cache\salt\minion" + +################################ Remove ################################## +if ( $Remove ) { + Write-Host "######################################################################" -ForegroundColor Cyan + Write-Host "Removing multi-minion" + Write-Host "Name: $Name" + Write-Host "Service Name: $service_name" + Write-Host "Root Dir: $RootDir" + Write-Host "######################################################################" -ForegroundColor Cyan + + # Stop Service + $service_object = Get-Service -Name $service_name -ErrorAction SilentlyContinue + if ( $service_object -and ($service_object.Status -ne "Stopped") ) { + Write-Host "Stopping service: " -NoNewline + Stop-Service -Name $service_name *> $null + $service_object.Refresh() + if ( $service_object.Status -eq "Stopped" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } + } + + # Remove Service + $service_object = Get-Service -Name $service_name -ErrorAction SilentlyContinue + if ( $service_object ) { + Write-Host "Removing service: " -NoNewline + & $ssm_bin remove $service_name confirm *> $null + $service_object = Get-Service -Name $service_name -ErrorAction SilentlyContinue + if ( !$service_object ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } + } + + # Remove Directory + if ( Test-Path -Path $RootDir ) { + Write-Host "Removing RootDir: " -NoNewline + Remove-Item -Path $RootDir -Force -Recurse + + if ( !(Test-Path -Path $RootDir) ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } + } + # Remind to delete keys from master + Write-Host "######################################################################" -ForegroundColor Cyan + Write-Host "Multi-Minion installed successfully" + Write-Host ">>>>> Don't forget to remove keys from the master <<<<<" + Write-Host "######################################################################" -ForegroundColor Cyan + exit 0 +} + +################################ Install ################################# +# We don't want to share config with the current running minion +if ( $RootDir.Trim("\") -eq $default_root_dir ) { + Write-Host "WARNING: RootDir can't be default Salt rootdir" -ForegroundColor Red + exit 1 +} + +# Make sure password is set if user is passed +if ( $User -and !$Password ) { + Write-Host "WARNING: You must pass a password when defining a user account" -ForegroundColor Red + exit 1 +} + +Write-Host "######################################################################" -ForegroundColor Cyan +Write-Host "Installing multi-minion" +Write-Host "Name: $Name" +Write-Host "Master: $Master" +Write-Host "Root Directory: $RootDir" +Write-Host "Create Service: $Service" +if ( $Service ) { + Write-Host "Service Account: $User" + Write-Host "Password: **********" + Write-Host "Service Name: $service_name" +} +Write-Host "######################################################################" -ForegroundColor Cyan + +# Create file_roots Directory Structure +if ( !( Test-Path -path "$RootDir" ) ) { + Write-Host "Creating RootDir: " -NoNewline + New-Item -Path "$RootDir" -Type Directory | Out-Null + if ( Test-Path -path "$RootDir" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } +} + +# Set permissions +if ( $User ) { + Write-Host "Setting Permissions: " -NoNewline + $acl = Get-Acl -Path "$RootDir" + $access_rule = New-Object System.Security.AccessControl.FileSystemAccessRule($User, "Modify", "Allow") + $acl.AddAccessRule($access_rule) + Set-Acl -Path "$RootDir" -AclObject $acl + + $found = $false + $acl = Get-Acl -Path "$RootDir" + $acl.Access | ForEach-Object { + if ( $_.IdentityReference.Value.Contains($User) ) { + $found = $true + } + } + if ( $found ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } +} + +# Child directories will inherit permissions from the parent +if ( !( Test-Path -path "$RootDir\conf" ) ) { + Write-Host "Creating config dir: " -NoNewline + New-Item -Path "$RootDir\conf" -Type Directory | Out-Null + if ( Test-Path -path "$RootDir\conf" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } +} +if ( !( Test-Path -path "$RootDir\conf\minion.d" ) ) { + Write-Host "Creating minion.d dir: " -NoNewline + New-Item -Path "$RootDir\conf\minion.d" -Type Directory | Out-Null + if ( Test-Path -path "$RootDir\conf\minion.d" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } +} +if ( !( Test-Path -path "$RootDir\conf\pki" ) ) { + Write-Host "Creating pki dir: " -NoNewline + New-Item -Path "$RootDir\conf\pki" -Type Directory | Out-Null + if ( Test-Path -path "$RootDir\conf\pki" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } +} +if ( !( Test-Path -path "$RootDir\var\log\salt" ) ) { + Write-Host "Creating log dir: " -NoNewline + New-Item -Path "$RootDir\var\log\salt" -Type Directory | Out-Null + if ( Test-Path -path "$RootDir\var\log\salt" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } +} +if ( !( Test-Path -path "$RootDir\var\run" ) ) { + Write-Host "Creating run dir: " -NoNewline + New-Item -Path "$RootDir\var\run" -Type Directory | Out-Null + if ( Test-Path -path "$RootDir\var\run" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } +} +if ( !( Test-Path -path "$cache_dir\extmods\grains" ) ) { + Write-Host "Creating extmods grains dir: " -NoNewline + New-Item -Path "$cache_dir\extmods\grains" -Type Directory | Out-Null + if ( Test-Path -path "$cache_dir\extmods\grains" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } +} +if ( !( Test-Path -path "$cache_dir\proc" ) ) { + Write-Host "Creating proc dir: " -NoNewline + New-Item -Path "$cache_dir\proc" -Type Directory | Out-Null + if ( Test-Path -path "$cache_dir\proc" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } +} + +# Write minion config +Write-Host "Writing minion config: " -NoNewline +Add-Content -Force -Path "$RootDir\conf\minion" -Value "master: $Master" +Add-Content -Force -Path "$RootDir\conf\minion" -Value "id: $Name" +Add-Content -Force -Path "$RootDir\conf\minion" -Value "root_dir: $RootDir" +Add-Content -Force -Path "$RootDir\conf\minion" -Value "log_file: $RootDir\var\log\salt\minion" +Add-Content -Force -Path "$RootDir\conf\minion" -Value "utils_dirs: $RootDir\var\cache\salt\minion\extmods\utils" +Add-Content -Force -Path "$RootDir\conf\minion" -Value "winrepo_dir: $RootDir\srv\salt\win\repo" +Add-Content -Force -Path "$RootDir\conf\minion" -Value "winrepo_dir_ng: $RootDir\srv\salt\win\repo-ng" + +Add-Content -Force -Path "$RootDir\conf\minion" -Value "file_roots:" +Add-Content -Force -Path "$RootDir\conf\minion" -Value " base:" +Add-Content -Force -Path "$RootDir\conf\minion" -Value " - $RootDir\srv\salt" +Add-Content -Force -Path "$RootDir\conf\minion" -Value " - $RootDir\srv\spm\salt" + +Add-Content -Force -Path "$RootDir\conf\minion" -Value "pillar_roots:" +Add-Content -Force -Path "$RootDir\conf\minion" -Value " base:" +Add-Content -Force -Path "$RootDir\conf\minion" -Value " - $RootDir\srv\pillar" +Add-Content -Force -Path "$RootDir\conf\minion" -Value " - $RootDir\srv\spm\pillar" + +Add-Content -Force -Path "$RootDir\conf\minion" -Value "thorium_roots:" +Add-Content -Force -Path "$RootDir\conf\minion" -Value " base:" +Add-Content -Force -Path "$RootDir\conf\minion" -Value " - $RootDir\srv\thorium" + +if ( Test-Path -path "$RootDir\conf\minion" ) { + Write-Host "Success" -ForegroundColor Green +} else { + Write-Host "Failed" -ForegroundColor Red + exit 1 +} + +if ( $Service ) { + # Register salt-minion service using SSM + Write-Host "Registering service $service_name`: " -NoNewline + & $ssm_bin install $service_name "$salt_bin" "-c """"$RootDir\conf"""" -l quiet" *> $null + & $ssm_bin set $service_name Description "Salt Minion $Name" *> $null + & $ssm_bin set $service_name Start SERVICE_AUTO_START *> $null + & $ssm_bin set $service_name AppStopMethodConsole 24000 *> $null + & $ssm_bin set $service_name AppStopMethodWindow 2000 *> $null + & $ssm_bin set $service_name AppRestartDelay 60000 *> $null + if ( $User -and $Password ) { + & $ssm_bin set $service_name ObjectName ".\$User" "$Password" *> $null + } + + $service_object = Get-Service -Name $service_name -ErrorAction SilentlyContinue + if ( $service_object ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } + + Write-Host "Starting service: " -NoNewline + Start-Service -Name $service_name + $service_object.Refresh() + if ( $service_object.Status -eq "Running" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } +} + +Write-Host "######################################################################" -ForegroundColor Cyan +Write-Host "Multi-Minion installed successfully" +Write-Host "Root Directory: $RootDir" +if ( $Service ) { + Write-Host "Service Name: $service_name" +} else { + Write-Host "To start the minion, run the following command:" + Write-Host "salt-minion -c `"$RootDir\conf`"" +} +Write-Host "######################################################################" -ForegroundColor Cyan From c016b46029a2a1f5677bf0ffc2c1bf6254e16ee4 Mon Sep 17 00:00:00 2001 From: Twangboy Date: Fri, 5 May 2023 09:29:53 -0600 Subject: [PATCH 04/53] Remove BOM to fix pre-commit --- pkg/windows/multi-minion.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/windows/multi-minion.ps1 b/pkg/windows/multi-minion.ps1 index 855a8fef2d5..dfd68fce573 100644 --- a/pkg/windows/multi-minion.ps1 +++ b/pkg/windows/multi-minion.ps1 @@ -1,4 +1,4 @@ -<# +<# .SYNOPSIS Script for setting up an additional salt-minion on a machine with Salt installed From 2f07a4e3af1c01e7a7f4a56b8f6638d031d93e1e Mon Sep 17 00:00:00 2001 From: Twangboy Date: Fri, 5 May 2023 10:39:50 -0600 Subject: [PATCH 05/53] Fix name of cmd script from master to minion --- pkg/windows/{multi-master.cmd => multi-minion.cmd} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename pkg/windows/{multi-master.cmd => multi-minion.cmd} (87%) diff --git a/pkg/windows/multi-master.cmd b/pkg/windows/multi-minion.cmd similarity index 87% rename from pkg/windows/multi-master.cmd rename to pkg/windows/multi-minion.cmd index aa8dbc72e45..2ec0e2ea578 100644 --- a/pkg/windows/multi-master.cmd +++ b/pkg/windows/multi-minion.cmd @@ -1,3 +1,3 @@ @ echo off Set "CurDir=%~dp0" -PowerShell -ExecutionPolicy RemoteSigned -File "%CurDir%\multi-master.ps1" %* +PowerShell -ExecutionPolicy RemoteSigned -File "%CurDir%\multi-minion.ps1" %* From 7a802fbb6ecd9e1a5d93a655a111a84bd88986ef Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 5 Jun 2023 16:49:18 -0600 Subject: [PATCH 06/53] Add scripts to installer, clean msi files --- pkg/windows/clean.ps1 | 27 +++++++++++++++++++++++++++ pkg/windows/multi-minion.cmd | 2 ++ pkg/windows/multi-minion.ps1 | 6 +++--- pkg/windows/prep_salt.ps1 | 20 ++++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/pkg/windows/clean.ps1 b/pkg/windows/clean.ps1 index 7d2234ad6b8..466cf812dcc 100644 --- a/pkg/windows/clean.ps1 +++ b/pkg/windows/clean.ps1 @@ -140,6 +140,33 @@ if ( Test-Path -Path "$RELENV_DIR" ) { } } +#------------------------------------------------------------------------------- +# Remove MSI build files +#------------------------------------------------------------------------------- +$files = @( + "msi/CustomAction01/CustomAction01.CA.dll", + "msi/CustomAction01/CustomAction01.dll", + "msi/CustomAction01/CustomAction01.pdb", + "msi/Product-discovered-files-config.wixobj", + "msi/Product-discovered-files-config.wxs", + "msi/Product-discovered-files-x64.wixobj", + "msi/Product-discovered-files-x64.wxs", + "msi/Product.wixobj" +) +$files | ForEach-Object { + if ( Test-Path -Path "$SCRIPT_DIR\$_" ) { + # Use .net, the powershell function is asynchronous + Write-Host "Removing $_`: " -NoNewline + [System.IO.File]::Delete("$SCRIPT_DIR\$_") + if ( ! (Test-Path -Path "$SCRIPT_DIR\$_") ) { + Write-Result "Success" -ForegroundColor Green + } else { + Write-Result "Failed" -ForegroundColor Red + exit 1 + } + } +} + #------------------------------------------------------------------------------- # Script Completed #------------------------------------------------------------------------------- diff --git a/pkg/windows/multi-minion.cmd b/pkg/windows/multi-minion.cmd index 2ec0e2ea578..3142158b469 100644 --- a/pkg/windows/multi-minion.cmd +++ b/pkg/windows/multi-minion.cmd @@ -1,3 +1,5 @@ +:: This is a helper script for multi-minion.ps1. +:: See multi-minion.ps1 for documentation @ echo off Set "CurDir=%~dp0" PowerShell -ExecutionPolicy RemoteSigned -File "%CurDir%\multi-minion.ps1" %* diff --git a/pkg/windows/multi-minion.ps1 b/pkg/windows/multi-minion.ps1 index dfd68fce573..c5f7977afaa 100644 --- a/pkg/windows/multi-minion.ps1 +++ b/pkg/windows/multi-minion.ps1 @@ -4,11 +4,11 @@ Script for setting up an additional salt-minion on a machine with Salt installed .DESCRIPTION This script will install an additional minion on a machine that already has a -Salt installtion using one of the Salt packages. It will set up the directory +Salt installation using one of the Salt packages. It will set up the directory structure required by Salt. It will also lay down a minion config to be used by the Salt minion. Additionaly, this script will install and start a Salt -minion service that uses the root_dir and minion config. You can also pass the -name of a service account to be used by the service. +minion service that uses the root_dir specified in the minion config. You can +also pass the name of a service account to be used by the service. You can also remove the multiminion setup with this script. diff --git a/pkg/windows/prep_salt.ps1 b/pkg/windows/prep_salt.ps1 index a3ee01a36d3..21ce25daaa6 100644 --- a/pkg/windows/prep_salt.ps1 +++ b/pkg/windows/prep_salt.ps1 @@ -165,6 +165,25 @@ if ( ! (Test-Path -Path "$BUILD_DIR\ssm.exe") ) { } } +# Copy the multiminion scripts to the Build directory +$scripts = @( + "multi-minion.cmd", + "multi-minion.ps1" +) +$scripts | ForEach-Object { + if (!(Test-Path -Path "$BUILD_DIR\$_")) { + Write-Host "Copying $_ to the Build directory: " -NoNewline + Copy-Item -Path "$SCRIPT_DIR\$_" -Destination "$BUILD_DIR\$_" + if (Test-Path -Path "$BUILD_DIR\$_") { + Write-Result "Success" -ForegroundColor Green + } else { + Write-Result "Failed" -ForegroundColor Red + exit 1 + } + } +} + +# Copy VCRedist 2013 to the prereqs directory New-Item -Path $PREREQ_DIR -ItemType Directory | Out-Null Write-Host "Copying VCRedist 2013 $ARCH_X to prereqs: " -NoNewline $file = "vcredist_$ARCH_X`_2013.exe" @@ -176,6 +195,7 @@ if ( Test-Path -Path "$PREREQ_DIR\$file" ) { exit 1 } +# Copy Universal C Runtimes to the prereqs directory Write-Host "Copying Universal C Runtimes $ARCH_X to prereqs: " -NoNewline $file = "ucrt_$ARCH_X.zip" Invoke-WebRequest -Uri "$SALT_DEP_URL/$file" -OutFile "$PREREQ_DIR\$file" From b0d14780fa908a7654357129d3c5c5820cd9a9f5 Mon Sep 17 00:00:00 2001 From: twangboy Date: Sun, 11 Jun 2023 13:01:35 -0600 Subject: [PATCH 07/53] Add docs, don't run as service --- doc/topics/windows/multi-minion.rst | 165 ++++++++++++ pkg/windows/multi-minion.ps1 | 386 ++++++++++++++-------------- 2 files changed, 360 insertions(+), 191 deletions(-) create mode 100644 doc/topics/windows/multi-minion.rst diff --git a/doc/topics/windows/multi-minion.rst b/doc/topics/windows/multi-minion.rst new file mode 100644 index 00000000000..292f322e4c7 --- /dev/null +++ b/doc/topics/windows/multi-minion.rst @@ -0,0 +1,165 @@ +============================= +Multi-Minion Setup on Windows +============================= + +There may be a scenario where having a minion running in the context of the +current logged in user would be useful. For example, the normal minion running +under the service account would perform machine-wide, administrative tasks. The +minion runing under the user context could be launched when the user logs in +and would be able to perform configuration tasks as if it were the user itself. +This would be useful for setting user registry settings, for example. + +The steps required to do this are as follows: + +1. Create new root_dir +2. Set root_dir permissions +3. Create directory structure +4. Write minion config + +We will now go through each of these steps in detail. + +.. note:: + + We have created a powershell script that will configure an additional minion + on the system for you. It can be found in the root of the Salt installation. + The script is named ``multi-minion.ps1``. You can get help on how to use the + script by running the following in a PowerShell prompt: + + ``Get-Help .\multi-minion.ps1 -Detailed`` + +1. Create new ``root_dir`` +-------------------------- + +The minion requires a root directory to store config, cache, logs, etc. The user +must have full permissions to this directory. The easiest way to do this is to +put the ``root_dir`` in the Local AppData directory (``$env:LocalAppData``). + +.. code-block:: powershell + + New-Item -Path "$env:LocalAppData\Salt Project\Salt" -Type Directory + +2. Set ``root_dir`` permissions +------------------------------- + +The user running Salt requires full access to the ``root_dir``. If you have +placed the root_dir in a location that the user does not have access to, you'll +need to give the user full permissions to that directory. + +.. code-block:: powershell + + $RootDir = "" + $User = "" + $acl = Get-Acl -Path "$RootDir" + $access_rule = New-Object System.Security.AccessControl.FileSystemAccessRule($User, "Modify", "Allow") + $acl.AddAccessRule($access_rule) + Set-Acl -Path "$RootDir" -AclObject $acl + +3. Create directory structure +----------------------------- + +Salt expects a certain directory structure to be present to avoid unnecessary +messages in the logs. This is usually handled by the installer. Since we're +running our own instance, we need to do it. Make sure the following directories +are present: + + - root_dir\\conf\\minion.d + - root_dir\\conf\\pki + - root_dir\\var\\log\\salt + - root_dir\\var\\run + - root_dir\\var\\cache\\salt\\minion\\extmods\\grains + - root_dir\\var\\cache\\salt\\minion\\proc + +.. code-block:: powershell + + $RootDir = "" + $cache_dir = "$RootDir\var\cache\salt\minion" + New-Item -Path "$RootDir\conf" -Type Directory + New-Item -Path "$RootDir\conf\minion.d" -Type Directory + New-Item -Path "$RootDir\conf\pki" -Type Directory + New-Item -Path "$RootDir\var\log\salt" -Type Directory + New-Item -Path "$RootDir\var\run" -Type Directory + New-Item -Path "$cache_dir\extmods\grains" -Type Directory + New-Item -Path "$cache_dir\proc" -Type Directory + +4. Write minion config +---------------------- + +The minion will need is own config, separate from the system minion config. This +config tells the minion where everything is as well as defines the master and +minion id. Create a minion config file named minion in the conf directory. + +.. code-block:: powershell + + New-Item -Path "$env:LocalAppData\Salt Project\Salt\conf\minion" -Type File + +Make sure the config file has at least the following contents: + +.. code-block:: yaml + + master: + id: + + root_dir: + log_file: \val\log\salt\minion + utils_dirs: + - \var\cache\salt\minion\extmods + winrepo_dir: \srv\salt\win\repo + winrepo_dir_ng: \srv\salt\win\repo-ng + + file_roots: + base: + - \srv\salt + - \srv\spm\salt + + pillar_roots: + base: + - \srv\pillar + - \srv\spm\pillar + + thorium_roots: + base: + - \srv\thorium + +Run the minion +-------------- + +Everything is now set up to run the minion. You can start the minion as you +would normally, but you need to specify the full path to the config file you +created above. + +.. code-block:: powershell + + salt-minion.exe -c \conf + +Register the minion as a service +-------------------------------- + +You can also register the minion as a service, but you need to understand the +implications of doing so. + +- You will need to have administrator privileges to register this minion service +- You will need the password to the user account that will be running the minion +- If the user password changes, you will have to update the service definition + to reflect the new password +- The minion will run all the time under the user context, whether that user is + logged in or not +- This requires great trust from the user as the minion will be able to perform + operations under the user's name without the user knowing, whether they are + logged in or not +- If you decide to run the new minion under the Local System account, it might + as well just be a normal minion +- The helper script does not support registering the 2nd minion as a service + +To register the minion as a service, use the ``ssm.exe`` binary that came with +the Salt installation. Run the following commands, replacing ````, +````, ````, and ```` as necessary: + +.. code-block:: powershell + + ssm.exe install "salt-minion.exe" "-c `"\conf`" -l quiet" + ssm.exe set Description "Salt Minion " + ssm.exe set Start SERVICE_AUTO_START + ssm.exe set AppStopMethodConsole 24000 + ssm.exe set AppStopMethodWindow 2000 + ssm.exe set AppRestartDelay 60000 + ssm.exe set ObjectName ".\" "" diff --git a/pkg/windows/multi-minion.ps1 b/pkg/windows/multi-minion.ps1 index c5f7977afaa..c4adfc4d17d 100644 --- a/pkg/windows/multi-minion.ps1 +++ b/pkg/windows/multi-minion.ps1 @@ -3,140 +3,160 @@ Script for setting up an additional salt-minion on a machine with Salt installed .DESCRIPTION -This script will install an additional minion on a machine that already has a +This script will configure an additional minion on a machine that already has a Salt installation using one of the Salt packages. It will set up the directory structure required by Salt. It will also lay down a minion config to be used -by the Salt minion. Additionaly, this script will install and start a Salt -minion service that uses the root_dir specified in the minion config. You can -also pass the name of a service account to be used by the service. +by the Salt minion. Additionaly, this script can start the new minion in a +hidden window. You can also remove the multiminion setup with this script. -This script should be run with Administrator privileges +This script does not need to be run with Administrator privileges -The following example will install a service named `salt-minion-mm10` that -starts with the LOCALSYSTEM account. It is the `-s` parameter that creates the -service: +If a minion that was configured with this script is already running, the script +will exit. + +The following example will set up a minion for the current logged in account. It +configures the minion to connect to the master at 192.168.0.10 .EXAMPLE -PS>multi-minion.ps1 -Name mm10 -s +PS>multi-minion.ps1 -Master 192.168.0.10 +PS>multi-minion.ps1 -m 192.168.0.10 -The following example will install a service that starts with a user named -mmuser: +The following example will set up a minion for the current logged in account. It +configures the minion to connect to the master at 192.168.0.10. It will also +prefix the minion id with `spongebob` .EXAMPLE -PS>multi-minion.ps1 -Name mm10 -s -m 192.168.0.10 -u mmuser -p secretword +PS>multi-minion.ps1 -Master 192.168.0.10 -Prefix spongebob +PS>multi-minion.ps1 -m 192.168.0.10 -p spongebob -The following example will set up config for minion that can be run in the -background under a user account. Notice the command does not have the `-s` -parameter: +The following example will set up a minion for the current logged in account. It +configures the minion to connect to the master at 192.168.0.10. It will also +start the minion in a hidden window: .EXAMPLE -PS>multi-minion.ps1 -Name mm10 -m 192.168.0.10 +PS>multi-minion.ps1 -Master 192.168.0.10 -Start +PS>multi-minion.ps1 -m 192.168.0.10 -s -The following example will remove a multiminion that has been installed with -this script: +The following example will remove a multiminion for the current running account: .EXAMPLE -PS>multi-minion.ps1 -Name mm10 -d +PS>multi-minion.ps1 -Delete +PS>multi-minion.ps1 -d #> [CmdletBinding()] param( - [Parameter(Mandatory=$true)] - [Alias("n")] - # The name used to create the service and root_dir. This is the only - # required parameter - [String] $Name, - [Parameter(Mandatory=$false)] [Alias("m")] # The master to connect to. This can be an ip address or an fqdn. Default # is salt [String] $Master = "salt", - [Parameter(Mandatory=$false)] - [Alias("r")] - # The root dir to place the minion config and directory structure. The - # default is %PROGRAMDATA%\Salt Project\Salt-$Name - [String] $RootDir = "$env:ProgramData\Salt Project\Salt-$Name", - - [Parameter(Mandatory=$false)] - [Alias("u")] - # User account to run the service under. The user account must be present on - # the system. The default is to use the LOCALSYSTEM account - [String] $User, - [Parameter(Mandatory=$false)] [Alias("p")] - # The password to the user account. Required if User is passed. We should - # probably figure out how to make this more secure - [String] $Password, + # The prefix to the minion id to differentiate it from the installed system + # minion. The default is $env:COMPUTERNAME. It might be helpful to use the + # minion id of the System minion if you know it + [String] $Prefix = "$env:COMPUTERNAME", [Parameter(Mandatory=$false)] [Alias("s")] - # Set this switch to install the service. Default is to not install the - # service - [Switch] $Service, + # Start the minion in the background + [Switch] $Start, + + [Parameter(Mandatory=$false)] + [Alias("l")] + [ValidateSet( + "all", + "garbage", + "trace", + "debug", + "profile", + "info", + "warning", + "error", + "critical", + "quiet" + )] + # Start the minion in the background + [String] $LogLevel = "warning", [Parameter(Mandatory=$false)] [Alias("d")] - # Remove the specified multi-minion. All other parameters are ignored + # Remove the multi-minion in the current account. All other parameters are + # ignored [Switch] $Remove ) ########################### Script Variables ############################# -$ssm_bin = "$env:ProgramFiles\Salt Project\Salt\ssm.exe" +$user_name = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name.Split("\")[-1].ToLower() $salt_bin = "$env:ProgramFiles\Salt Project\Salt\salt-minion.exe" -$service_name = "salt-minion-$($Name.ToLower())" -$default_root_dir = Resolve-Path -Path "$env:ProgramData\Salt Project\Salt" -$cache_dir = "$RootDir\var\cache\salt\minion" +$root_dir = "$env:LocalAppData\Salt Project\Salt" +$cache_dir = "$root_dir\var\cache\salt\minion" +$minion_id = "$Prefix-$user_name" + +########################### Script Functions ############################# +function Test-FileLock { + param ( + [parameter(Mandatory=$true)] + # The path to the file to check + [string]$Path + ) + if ((Test-Path -Path $Path) -eq $false) { + return $false + } + $oFile = New-Object System.IO.FileInfo $Path + try { + $oStream = $oFile.Open([System.IO.FileMode]::Open, [System.IO.FileAccess]::ReadWrite, [System.IO.FileShare]::None) + if ($oStream) { + $oStream.Close() + } + return $false + } catch { + # file is locked by a process. + return $true + } +} ################################ Remove ################################## if ( $Remove ) { Write-Host "######################################################################" -ForegroundColor Cyan Write-Host "Removing multi-minion" - Write-Host "Name: $Name" - Write-Host "Service Name: $service_name" - Write-Host "Root Dir: $RootDir" + Write-Host "Root Dir: $root_dir" Write-Host "######################################################################" -ForegroundColor Cyan - # Stop Service - $service_object = Get-Service -Name $service_name -ErrorAction SilentlyContinue - if ( $service_object -and ($service_object.Status -ne "Stopped") ) { - Write-Host "Stopping service: " -NoNewline - Stop-Service -Name $service_name *> $null - $service_object.Refresh() - if ( $service_object.Status -eq "Stopped" ) { - Write-Host "Success" -ForegroundColor Green - } else { - Write-Host "Failed" -ForegroundColor Red - exit 1 + # Stop salt-minion service if running + $processes = Get-WmiObject win32_process -filter "name like '%salt-minion%'" | Select-Object commandline,handle + $processes | ForEach-Object { + if ( $_.commandline -like "*$root_dir*" ) { + Write-Host "Killing process: " -NoNewline + $process = Get-Process -Id $_.handle + $process.Kill() + if ( $process.HasExited ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } } } - # Remove Service - $service_object = Get-Service -Name $service_name -ErrorAction SilentlyContinue - if ( $service_object ) { - Write-Host "Removing service: " -NoNewline - & $ssm_bin remove $service_name confirm *> $null - $service_object = Get-Service -Name $service_name -ErrorAction SilentlyContinue - if ( !$service_object ) { - Write-Host "Success" -ForegroundColor Green - } else { - Write-Host "Failed" -ForegroundColor Red - exit 1 - } + # Check for locked log file + # The log file will be locked until the running process releases it + while (Test-FileLock -Path "$root_dir\var\log\salt\minion") { + Start-Sleep -Seconds 1 } # Remove Directory - if ( Test-Path -Path $RootDir ) { - Write-Host "Removing RootDir: " -NoNewline - Remove-Item -Path $RootDir -Force -Recurse + if ( Test-Path -Path $root_dir) { + Write-Host "Removing Root Dir: " -NoNewline + Remove-Item -Path $root_dir -Force -Recurse - if ( !(Test-Path -Path $RootDir) ) { + if ( !(Test-Path -Path $root_dir) ) { Write-Host "Success" -ForegroundColor Green } else { Write-Host "Failed" -ForegroundColor Red @@ -145,43 +165,44 @@ if ( $Remove ) { } # Remind to delete keys from master Write-Host "######################################################################" -ForegroundColor Cyan - Write-Host "Multi-Minion installed successfully" + Write-Host "Multi-Minion successfully removed" Write-Host ">>>>> Don't forget to remove keys from the master <<<<<" Write-Host "######################################################################" -ForegroundColor Cyan exit 0 } -################################ Install ################################# -# We don't want to share config with the current running minion -if ( $RootDir.Trim("\") -eq $default_root_dir ) { - Write-Host "WARNING: RootDir can't be default Salt rootdir" -ForegroundColor Red - exit 1 +################################ EXISTING CHECK ################################ + +# See there is already a running minion +$running = $false +$processes = Get-WmiObject win32_process -filter "name like '%salt-minion%'" | Select-Object commandline,handle +$processes | ForEach-Object { + if ( $_.commandline -like "*$root_dir*" ) { + $running = $true + } +} +if ( $running ) { + Write-Host "######################################################################" -ForegroundColor Cyan + Write-Host "Multi-Minion" + Write-Host "A minion is already running for this user" + Write-Host "######################################################################" -ForegroundColor Cyan + exit 0 } -# Make sure password is set if user is passed -if ( $User -and !$Password ) { - Write-Host "WARNING: You must pass a password when defining a user account" -ForegroundColor Red - exit 1 -} +################################### INSTALL #################################### Write-Host "######################################################################" -ForegroundColor Cyan -Write-Host "Installing multi-minion" -Write-Host "Name: $Name" -Write-Host "Master: $Master" -Write-Host "Root Directory: $RootDir" -Write-Host "Create Service: $Service" -if ( $Service ) { - Write-Host "Service Account: $User" - Write-Host "Password: **********" - Write-Host "Service Name: $service_name" -} +Write-Host "Installing Multi-Minion" +Write-Host "Master: $Master" +Write-Host "Minion ID: $minion_id" +Write-Host "Root Directory: $root_dir" Write-Host "######################################################################" -ForegroundColor Cyan -# Create file_roots Directory Structure -if ( !( Test-Path -path "$RootDir" ) ) { - Write-Host "Creating RootDir: " -NoNewline - New-Item -Path "$RootDir" -Type Directory | Out-Null - if ( Test-Path -path "$RootDir" ) { +# Create Root Directory Structure +if ( !( Test-Path -path "$root_dir" ) ) { + Write-Host "Creating Root Dir: " -NoNewline + New-Item -Path "$root_dir" -Type Directory | Out-Null + if ( Test-Path -path "$root_dir" ) { Write-Host "Success" -ForegroundColor Green } else { Write-Host "Failed" -ForegroundColor Red @@ -189,80 +210,67 @@ if ( !( Test-Path -path "$RootDir" ) ) { } } -# Set permissions -if ( $User ) { - Write-Host "Setting Permissions: " -NoNewline - $acl = Get-Acl -Path "$RootDir" - $access_rule = New-Object System.Security.AccessControl.FileSystemAccessRule($User, "Modify", "Allow") - $acl.AddAccessRule($access_rule) - Set-Acl -Path "$RootDir" -AclObject $acl - - $found = $false - $acl = Get-Acl -Path "$RootDir" - $acl.Access | ForEach-Object { - if ( $_.IdentityReference.Value.Contains($User) ) { - $found = $true - } - } - if ( $found ) { - Write-Host "Success" -ForegroundColor Green - } else { - Write-Host "Failed" -ForegroundColor Red - exit 1 - } -} - -# Child directories will inherit permissions from the parent -if ( !( Test-Path -path "$RootDir\conf" ) ) { +# Config dir +if ( !( Test-Path -path "$root_dir\conf" ) ) { Write-Host "Creating config dir: " -NoNewline - New-Item -Path "$RootDir\conf" -Type Directory | Out-Null - if ( Test-Path -path "$RootDir\conf" ) { + New-Item -Path "$root_dir\conf" -Type Directory | Out-Null + if ( Test-Path -path "$root_dir\conf" ) { Write-Host "Success" -ForegroundColor Green } else { Write-Host "Failed" -ForegroundColor Red exit 1 } } -if ( !( Test-Path -path "$RootDir\conf\minion.d" ) ) { + +# Minion.d dir +if ( !( Test-Path -path "$root_dir\conf\minion.d" ) ) { Write-Host "Creating minion.d dir: " -NoNewline - New-Item -Path "$RootDir\conf\minion.d" -Type Directory | Out-Null - if ( Test-Path -path "$RootDir\conf\minion.d" ) { + New-Item -Path "$root_dir\conf\minion.d" -Type Directory | Out-Null + if ( Test-Path -path "$root_dir\conf\minion.d" ) { Write-Host "Success" -ForegroundColor Green } else { Write-Host "Failed" -ForegroundColor Red exit 1 } } -if ( !( Test-Path -path "$RootDir\conf\pki" ) ) { + +# PKI dir +if ( !( Test-Path -path "$root_dir\conf\pki" ) ) { Write-Host "Creating pki dir: " -NoNewline - New-Item -Path "$RootDir\conf\pki" -Type Directory | Out-Null - if ( Test-Path -path "$RootDir\conf\pki" ) { + New-Item -Path "$root_dir\conf\pki" -Type Directory | Out-Null + if ( Test-Path -path "$root_dir\conf\pki" ) { Write-Host "Success" -ForegroundColor Green } else { Write-Host "Failed" -ForegroundColor Red exit 1 } } -if ( !( Test-Path -path "$RootDir\var\log\salt" ) ) { + +# Log dir +if ( !( Test-Path -path "$root_dir\var\log\salt" ) ) { Write-Host "Creating log dir: " -NoNewline - New-Item -Path "$RootDir\var\log\salt" -Type Directory | Out-Null - if ( Test-Path -path "$RootDir\var\log\salt" ) { + New-Item -Path "$root_dir\var\log\salt" -Type Directory | Out-Null + if ( Test-Path -path "$root_dir\var\log\salt" ) { Write-Host "Success" -ForegroundColor Green } else { Write-Host "Failed" -ForegroundColor Red exit 1 } } -if ( !( Test-Path -path "$RootDir\var\run" ) ) { + +# Run dir +if ( !( Test-Path -path "$root_dir\var\run" ) ) { Write-Host "Creating run dir: " -NoNewline - New-Item -Path "$RootDir\var\run" -Type Directory | Out-Null - if ( Test-Path -path "$RootDir\var\run" ) { + New-Item -Path "$root_dir\var\run" -Type Directory | Out-Null + if ( Test-Path -path "$root_dir\var\run" ) { Write-Host "Success" -ForegroundColor Green } else { Write-Host "Failed" -ForegroundColor Red exit 1 } } + +# Extmods grains dir if ( !( Test-Path -path "$cache_dir\extmods\grains" ) ) { Write-Host "Creating extmods grains dir: " -NoNewline New-Item -Path "$cache_dir\extmods\grains" -Type Directory | Out-Null @@ -273,6 +281,8 @@ if ( !( Test-Path -path "$cache_dir\extmods\grains" ) ) { exit 1 } } + +# Proc dir if ( !( Test-Path -path "$cache_dir\proc" ) ) { Write-Host "Creating proc dir: " -NoNewline New-Item -Path "$cache_dir\proc" -Type Directory | Out-Null @@ -286,60 +296,53 @@ if ( !( Test-Path -path "$cache_dir\proc" ) ) { # Write minion config Write-Host "Writing minion config: " -NoNewline -Add-Content -Force -Path "$RootDir\conf\minion" -Value "master: $Master" -Add-Content -Force -Path "$RootDir\conf\minion" -Value "id: $Name" -Add-Content -Force -Path "$RootDir\conf\minion" -Value "root_dir: $RootDir" -Add-Content -Force -Path "$RootDir\conf\minion" -Value "log_file: $RootDir\var\log\salt\minion" -Add-Content -Force -Path "$RootDir\conf\minion" -Value "utils_dirs: $RootDir\var\cache\salt\minion\extmods\utils" -Add-Content -Force -Path "$RootDir\conf\minion" -Value "winrepo_dir: $RootDir\srv\salt\win\repo" -Add-Content -Force -Path "$RootDir\conf\minion" -Value "winrepo_dir_ng: $RootDir\srv\salt\win\repo-ng" +Set-Content -Force -Path "$root_dir\conf\minion" -Value "master: $Master" +Add-Content -Force -Path "$root_dir\conf\minion" -Value "id: $minion_id" +Add-Content -Force -Path "$root_dir\conf\minion" -Value "root_dir: $root_dir" +Add-Content -Force -Path "$root_dir\conf\minion" -Value "log_file: $root_dir\var\log\salt\minion" +Add-Content -Force -Path "$root_dir\conf\minion" -Value "log_level_logfile: $LogLevel" -Add-Content -Force -Path "$RootDir\conf\minion" -Value "file_roots:" -Add-Content -Force -Path "$RootDir\conf\minion" -Value " base:" -Add-Content -Force -Path "$RootDir\conf\minion" -Value " - $RootDir\srv\salt" -Add-Content -Force -Path "$RootDir\conf\minion" -Value " - $RootDir\srv\spm\salt" +Add-Content -Force -Path "$root_dir\conf\minion" -Value "utils_dirs:" +Add-Content -Force -Path "$root_dir\conf\minion" -Value " - $root_dir\var\cache\salt\minion\extmods\utils" +Add-Content -Force -Path "$root_dir\conf\minion" -Value "winrepo_dir: $root_dir\srv\salt\win\repo" +Add-Content -Force -Path "$root_dir\conf\minion" -Value "winrepo_dir_ng: $root_dir\srv\salt\win\repo-ng" -Add-Content -Force -Path "$RootDir\conf\minion" -Value "pillar_roots:" -Add-Content -Force -Path "$RootDir\conf\minion" -Value " base:" -Add-Content -Force -Path "$RootDir\conf\minion" -Value " - $RootDir\srv\pillar" -Add-Content -Force -Path "$RootDir\conf\minion" -Value " - $RootDir\srv\spm\pillar" +Add-Content -Force -Path "$root_dir\conf\minion" -Value "file_roots:" +Add-Content -Force -Path "$root_dir\conf\minion" -Value " base:" +Add-Content -Force -Path "$root_dir\conf\minion" -Value " - $root_dir\srv\salt" +Add-Content -Force -Path "$root_dir\conf\minion" -Value " - $root_dir\srv\spm\salt" -Add-Content -Force -Path "$RootDir\conf\minion" -Value "thorium_roots:" -Add-Content -Force -Path "$RootDir\conf\minion" -Value " base:" -Add-Content -Force -Path "$RootDir\conf\minion" -Value " - $RootDir\srv\thorium" +Add-Content -Force -Path "$root_dir\conf\minion" -Value "pillar_roots:" +Add-Content -Force -Path "$root_dir\conf\minion" -Value " base:" +Add-Content -Force -Path "$root_dir\conf\minion" -Value " - $root_dir\srv\pillar" +Add-Content -Force -Path "$root_dir\conf\minion" -Value " - $root_dir\srv\spm\pillar" -if ( Test-Path -path "$RootDir\conf\minion" ) { +Add-Content -Force -Path "$root_dir\conf\minion" -Value "thorium_roots:" +Add-Content -Force -Path "$root_dir\conf\minion" -Value " base:" +Add-Content -Force -Path "$root_dir\conf\minion" -Value " - $root_dir\srv\thorium" + +if ( Test-Path -path "$root_dir\conf\minion" ) { Write-Host "Success" -ForegroundColor Green } else { Write-Host "Failed" -ForegroundColor Red exit 1 } -if ( $Service ) { - # Register salt-minion service using SSM - Write-Host "Registering service $service_name`: " -NoNewline - & $ssm_bin install $service_name "$salt_bin" "-c """"$RootDir\conf"""" -l quiet" *> $null - & $ssm_bin set $service_name Description "Salt Minion $Name" *> $null - & $ssm_bin set $service_name Start SERVICE_AUTO_START *> $null - & $ssm_bin set $service_name AppStopMethodConsole 24000 *> $null - & $ssm_bin set $service_name AppStopMethodWindow 2000 *> $null - & $ssm_bin set $service_name AppRestartDelay 60000 *> $null - if ( $User -and $Password ) { - & $ssm_bin set $service_name ObjectName ".\$User" "$Password" *> $null +# Start the minion +if ( $Start ) { + Write-Host "Starting minion process: " -NoNewline + Start-Process -FilePath "$salt_bin" ` + -ArgumentList "-c","`"$root_dir\conf`"" ` + -WindowStyle Hidden + # Verify running minion + $running = $false + $processes = Get-WmiObject win32_process -filter "name like '%salt-minion%'" | Select-Object commandline,handle + $processes | ForEach-Object { + if ( $_.commandline -like "*$root_dir*" ) { + $running = $true + } } - - $service_object = Get-Service -Name $service_name -ErrorAction SilentlyContinue - if ( $service_object ) { - Write-Host "Success" -ForegroundColor Green - } else { - Write-Host "Failed" -ForegroundColor Red - exit 1 - } - - Write-Host "Starting service: " -NoNewline - Start-Service -Name $service_name - $service_object.Refresh() - if ( $service_object.Status -eq "Running" ) { + if ( $running ) { Write-Host "Success" -ForegroundColor Green } else { Write-Host "Failed" -ForegroundColor Red @@ -349,11 +352,12 @@ if ( $Service ) { Write-Host "######################################################################" -ForegroundColor Cyan Write-Host "Multi-Minion installed successfully" -Write-Host "Root Directory: $RootDir" -if ( $Service ) { - Write-Host "Service Name: $service_name" -} else { +if ( ! $Start ) { + Write-Host "" Write-Host "To start the minion, run the following command:" - Write-Host "salt-minion -c `"$RootDir\conf`"" + Write-Host "salt-minion -c `"$root_dir\conf`"" + Write-Host "" + Write-Host "To start the minion in the background, run the following command:" + Write-Host "Start-Process -FilePath salt-minion.exe -ArgumentList `"-c`",'`"$root_dir\conf`"' -WindowStyle Hidden" } Write-Host "######################################################################" -ForegroundColor Cyan From 71a8f79313b76ec0fdd7e1412600377858619eb7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Sun, 11 Jun 2023 13:05:43 -0600 Subject: [PATCH 08/53] Add changelog --- changelog/64439.added.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog/64439.added.md diff --git a/changelog/64439.added.md b/changelog/64439.added.md new file mode 100644 index 00000000000..93c3000643c --- /dev/null +++ b/changelog/64439.added.md @@ -0,0 +1,2 @@ +Added documentation on how to run a 2nd minion in a user context on Windows +Also created a script to automate setting up a 2nd minion From a404f9ec1a6d364edecc753c74d24a31eda76d14 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 12 Jun 2023 13:35:08 -0600 Subject: [PATCH 09/53] Fix spelling issues --- doc/topics/windows/multi-minion.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/topics/windows/multi-minion.rst b/doc/topics/windows/multi-minion.rst index 292f322e4c7..049cd05079f 100644 --- a/doc/topics/windows/multi-minion.rst +++ b/doc/topics/windows/multi-minion.rst @@ -3,11 +3,10 @@ Multi-Minion Setup on Windows ============================= There may be a scenario where having a minion running in the context of the -current logged in user would be useful. For example, the normal minion running +current, logged-in user would be useful. For example, the normal minion running under the service account would perform machine-wide, administrative tasks. The -minion runing under the user context could be launched when the user logs in +minion running under the user context could be launched when the user logs in and would be able to perform configuration tasks as if it were the user itself. -This would be useful for setting user registry settings, for example. The steps required to do this are as follows: @@ -86,7 +85,7 @@ are present: The minion will need is own config, separate from the system minion config. This config tells the minion where everything is as well as defines the master and -minion id. Create a minion config file named minion in the conf directory. +minion id. Create a minion config file named ``minion`` in the conf directory. .. code-block:: powershell From 901708cf0963da1a00a0f3dc169a3f6b0e625c59 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 12 Jun 2023 16:56:06 -0600 Subject: [PATCH 10/53] Add tests for multi-minion script --- pkg/tests/integration/test_multi_minion.py | 127 +++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 pkg/tests/integration/test_multi_minion.py diff --git a/pkg/tests/integration/test_multi_minion.py b/pkg/tests/integration/test_multi_minion.py new file mode 100644 index 00000000000..1eb3b70cbb3 --- /dev/null +++ b/pkg/tests/integration/test_multi_minion.py @@ -0,0 +1,127 @@ +import os +import pathlib +import subprocess + +import psutil +import pytest + +pytestmark = [ + pytest.mark.skip_unless_on_windows, +] + + +@pytest.fixture +def mm_script(install_salt): + yield install_salt.ssm_bin.parent / "multi-minion.ps1" + + +@pytest.fixture(scope="function") +def mm_conf(mm_script): + yield pathlib.Path(os.getenv("LocalAppData"), "Salt Project", "Salt", "conf") + subprocess.run( + ["powershell", "-c", str(mm_script), "-d"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + universal_newlines=True, + ) + + +def test_script_present(mm_script): + """ + Ensure the multi-minion.ps1 file is present in the root of the installation + """ + assert mm_script.exists() + + +def test_install(mm_script, mm_conf): + """ + Install a 2nd minion with default settings. Should create a minion config + file in Local AppData + """ + ret = subprocess.run( + ["powershell", "-c", str(mm_script)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + universal_newlines=True, + ) + assert ret.returncode == 0, ret.stderr + conf_file = mm_conf / "minion" + assert conf_file.exists() + assert conf_file.read_text().find("master: salt") > -1 + + +def test_install_master(mm_script, mm_conf): + """ + Install a 2nd minion and set the master to spongebob + """ + ret = subprocess.run( + ["powershell", "-c", str(mm_script), "-m", "spongebob"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + universal_newlines=True, + ) + assert ret.returncode == 0, ret.stderr + conf_file = mm_conf / "minion" + assert conf_file.exists() + assert conf_file.read_text().find("master: spongebob") > -1 + + +def test_install_prefix(mm_script, mm_conf): + """ + Install a 2nd minion and add a prefix to the minion id + """ + ret = subprocess.run( + ["powershell", "-c", str(mm_script), "-p", "squarepants"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + universal_newlines=True, + ) + assert ret.returncode == 0, ret.stderr + conf_file = mm_conf / "minion" + assert conf_file.exists() + assert conf_file.read_text().find("id: squarepants") > -1 + + +def test_install_log_level(mm_script, mm_conf): + """ + Install a 2nd minion and set the log level in the log file to debug + """ + ret = subprocess.run( + ["powershell", "-c", str(mm_script), "-l", "debug"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + universal_newlines=True, + ) + assert ret.returncode == 0, ret.stderr + conf_file = mm_conf / "minion" + assert conf_file.exists() + assert conf_file.read_text().find("log_level_logfile: debug") > -1 + + +def test_install_start(mm_script, mm_conf): + """ + Install a 2nd minion and start that minion in a hidden process + """ + ret = subprocess.run( + ["powershell", "-c", str(mm_script), "-s"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + universal_newlines=True, + ) + assert ret.returncode == 0, ret.stderr + conf_file = mm_conf / "minion" + assert conf_file.exists() + assert conf_file.read_text().find("master: salt") > -1 + + found = False + for p in psutil.process_iter(["cmdline", "name"]): + if p.info["name"] and p.info["name"] == "salt-minion.exe": + if f"{mm_conf}" in p.info["cmdline"]: + found = True + assert found is True From d516d1784042b88a62e6d9cf52bb05562d326b75 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 13 Jun 2023 10:17:42 -0600 Subject: [PATCH 11/53] Quote path to salt-bin --- pkg/windows/multi-minion.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/windows/multi-minion.ps1 b/pkg/windows/multi-minion.ps1 index c4adfc4d17d..838fbef1a3f 100644 --- a/pkg/windows/multi-minion.ps1 +++ b/pkg/windows/multi-minion.ps1 @@ -331,7 +331,7 @@ if ( Test-Path -path "$root_dir\conf\minion" ) { # Start the minion if ( $Start ) { Write-Host "Starting minion process: " -NoNewline - Start-Process -FilePath "$salt_bin" ` + Start-Process -FilePath "`"$salt_bin`"" ` -ArgumentList "-c","`"$root_dir\conf`"" ` -WindowStyle Hidden # Verify running minion From f036af2953836745e17f425da454445dcf3ea132 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 13 Jun 2023 11:42:57 -0600 Subject: [PATCH 12/53] Quote spaces in the script path --- pkg/tests/integration/test_multi_minion.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/tests/integration/test_multi_minion.py b/pkg/tests/integration/test_multi_minion.py index 1eb3b70cbb3..f4162958e6b 100644 --- a/pkg/tests/integration/test_multi_minion.py +++ b/pkg/tests/integration/test_multi_minion.py @@ -19,7 +19,7 @@ def mm_script(install_salt): def mm_conf(mm_script): yield pathlib.Path(os.getenv("LocalAppData"), "Salt Project", "Salt", "conf") subprocess.run( - ["powershell", "-c", str(mm_script), "-d"], + ["powershell", str(mm_script).replace(" ", "' '"), "-d"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, @@ -40,7 +40,7 @@ def test_install(mm_script, mm_conf): file in Local AppData """ ret = subprocess.run( - ["powershell", "-c", str(mm_script)], + ["powershell", str(mm_script).replace(" ", "' '")], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, @@ -57,7 +57,7 @@ def test_install_master(mm_script, mm_conf): Install a 2nd minion and set the master to spongebob """ ret = subprocess.run( - ["powershell", "-c", str(mm_script), "-m", "spongebob"], + ["powershell", str(mm_script).replace(" ", "' '"), "-m", "spongebob"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, @@ -74,7 +74,7 @@ def test_install_prefix(mm_script, mm_conf): Install a 2nd minion and add a prefix to the minion id """ ret = subprocess.run( - ["powershell", "-c", str(mm_script), "-p", "squarepants"], + ["powershell", str(mm_script).replace(" ", "' '"), "-p", "squarepants"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, @@ -91,7 +91,7 @@ def test_install_log_level(mm_script, mm_conf): Install a 2nd minion and set the log level in the log file to debug """ ret = subprocess.run( - ["powershell", "-c", str(mm_script), "-l", "debug"], + ["powershell", str(mm_script).replace(" ", "' '"), "-l", "debug"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, @@ -108,7 +108,7 @@ def test_install_start(mm_script, mm_conf): Install a 2nd minion and start that minion in a hidden process """ ret = subprocess.run( - ["powershell", "-c", str(mm_script), "-s"], + ["powershell", str(mm_script).replace(" ", "' '"), "-s"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, From e7832f772d5c2c84c8c027f9c1953969a5c63279 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 14 Jun 2023 22:47:08 -0600 Subject: [PATCH 13/53] Add suggested docs fixes --- changelog/64439.added.md | 3 +- doc/topics/windows/multi-minion.rst | 59 ++++++++++++---------- pkg/tests/integration/test_multi_minion.py | 10 ++-- pkg/windows/multi-minion.ps1 | 26 +++++----- 4 files changed, 52 insertions(+), 46 deletions(-) diff --git a/changelog/64439.added.md b/changelog/64439.added.md index 93c3000643c..a26b56f9cbc 100644 --- a/changelog/64439.added.md +++ b/changelog/64439.added.md @@ -1,2 +1 @@ -Added documentation on how to run a 2nd minion in a user context on Windows -Also created a script to automate setting up a 2nd minion +Added a script to automate setting up a 2nd minion in a user context on Windows diff --git a/doc/topics/windows/multi-minion.rst b/doc/topics/windows/multi-minion.rst index 049cd05079f..17f50e96259 100644 --- a/doc/topics/windows/multi-minion.rst +++ b/doc/topics/windows/multi-minion.rst @@ -1,5 +1,5 @@ ============================= -Multi-Minion Setup on Windows +Multi-minion setup on Windows ============================= There may be a scenario where having a minion running in the context of the @@ -14,18 +14,21 @@ The steps required to do this are as follows: 2. Set root_dir permissions 3. Create directory structure 4. Write minion config - -We will now go through each of these steps in detail. +5. Start the minion +6. Register the minion as a service (optional) .. note:: - We have created a powershell script that will configure an additional minion - on the system for you. It can be found in the root of the Salt installation. - The script is named ``multi-minion.ps1``. You can get help on how to use the - script by running the following in a PowerShell prompt: + The Salt Project has created a powershell script that will configure an + additional minion on the system for you. It can be found in the root of the + Salt installation. The script is named ``multi-minion.ps1``. You can get + help on how to use the script by running the following in a PowerShell + prompt: ``Get-Help .\multi-minion.ps1 -Detailed`` +The following guide explains these steps in more detail. + 1. Create new ``root_dir`` -------------------------- @@ -42,7 +45,8 @@ put the ``root_dir`` in the Local AppData directory (``$env:LocalAppData``). The user running Salt requires full access to the ``root_dir``. If you have placed the root_dir in a location that the user does not have access to, you'll -need to give the user full permissions to that directory. +need to give the user full permissions to that directory. Replace the + in this example with your own configuration information. .. code-block:: powershell @@ -57,9 +61,9 @@ need to give the user full permissions to that directory. ----------------------------- Salt expects a certain directory structure to be present to avoid unnecessary -messages in the logs. This is usually handled by the installer. Since we're -running our own instance, we need to do it. Make sure the following directories -are present: +messages in the logs. This is usually handled by the installer. Since you're +running your own instance, you need to do it. Make sure the following +directories are present: - root_dir\\conf\\minion.d - root_dir\\conf\\pki @@ -83,9 +87,10 @@ are present: 4. Write minion config ---------------------- -The minion will need is own config, separate from the system minion config. This -config tells the minion where everything is as well as defines the master and -minion id. Create a minion config file named ``minion`` in the conf directory. +The minion will need its own config, separate from the system minion config. +This config tells the minion where everything is located in the file structure +and also defines the master and minion id. Create a minion config file named +``minion`` in the conf directory. .. code-block:: powershell @@ -119,8 +124,8 @@ Make sure the config file has at least the following contents: base: - \srv\thorium -Run the minion --------------- +5. Run the minion +----------------- Everything is now set up to run the minion. You can start the minion as you would normally, but you need to specify the full path to the config file you @@ -130,24 +135,26 @@ created above. salt-minion.exe -c \conf -Register the minion as a service --------------------------------- +6. Register the minion as a service (optional) +---------------------------------------------- You can also register the minion as a service, but you need to understand the implications of doing so. -- You will need to have administrator privileges to register this minion service -- You will need the password to the user account that will be running the minion +- You will need to have administrator privileges to register this minion + service. +- You will need the password to the user account that will be running the + minion. - If the user password changes, you will have to update the service definition - to reflect the new password -- The minion will run all the time under the user context, whether that user is - logged in or not + to reflect the new password. +- The minion runs all the time under the user context, whether that user is + logged in or not. - This requires great trust from the user as the minion will be able to perform operations under the user's name without the user knowing, whether they are - logged in or not + logged in or not. - If you decide to run the new minion under the Local System account, it might - as well just be a normal minion -- The helper script does not support registering the 2nd minion as a service + as well just be a normal minion. +- The helper script does not support registering the second minion as a service. To register the minion as a service, use the ``ssm.exe`` binary that came with the Salt installation. Run the following commands, replacing ````, diff --git a/pkg/tests/integration/test_multi_minion.py b/pkg/tests/integration/test_multi_minion.py index f4162958e6b..13d64f31f6e 100644 --- a/pkg/tests/integration/test_multi_minion.py +++ b/pkg/tests/integration/test_multi_minion.py @@ -36,7 +36,7 @@ def test_script_present(mm_script): def test_install(mm_script, mm_conf): """ - Install a 2nd minion with default settings. Should create a minion config + Install a second minion with default settings. Should create a minion config file in Local AppData """ ret = subprocess.run( @@ -54,7 +54,7 @@ def test_install(mm_script, mm_conf): def test_install_master(mm_script, mm_conf): """ - Install a 2nd minion and set the master to spongebob + Install a second minion and set the master to spongebob """ ret = subprocess.run( ["powershell", str(mm_script).replace(" ", "' '"), "-m", "spongebob"], @@ -71,7 +71,7 @@ def test_install_master(mm_script, mm_conf): def test_install_prefix(mm_script, mm_conf): """ - Install a 2nd minion and add a prefix to the minion id + Install a second minion and add a prefix to the minion id """ ret = subprocess.run( ["powershell", str(mm_script).replace(" ", "' '"), "-p", "squarepants"], @@ -88,7 +88,7 @@ def test_install_prefix(mm_script, mm_conf): def test_install_log_level(mm_script, mm_conf): """ - Install a 2nd minion and set the log level in the log file to debug + Install a second minion and set the log level in the log file to debug """ ret = subprocess.run( ["powershell", str(mm_script).replace(" ", "' '"), "-l", "debug"], @@ -105,7 +105,7 @@ def test_install_log_level(mm_script, mm_conf): def test_install_start(mm_script, mm_conf): """ - Install a 2nd minion and start that minion in a hidden process + Install a second minion and start that minion in a hidden process """ ret = subprocess.run( ["powershell", str(mm_script).replace(" ", "' '"), "-s"], diff --git a/pkg/windows/multi-minion.ps1 b/pkg/windows/multi-minion.ps1 index 838fbef1a3f..8ad709c04cc 100644 --- a/pkg/windows/multi-minion.ps1 +++ b/pkg/windows/multi-minion.ps1 @@ -3,9 +3,9 @@ Script for setting up an additional salt-minion on a machine with Salt installed .DESCRIPTION -This script will configure an additional minion on a machine that already has a -Salt installation using one of the Salt packages. It will set up the directory -structure required by Salt. It will also lay down a minion config to be used +This script configures an additional minion on a machine that already has a Salt +installation using one of the Salt packages. It sets up the directory structure +required by Salt. It also lays down a minion config to be used by the Salt minion. Additionaly, this script can start the new minion in a hidden window. @@ -16,30 +16,30 @@ This script does not need to be run with Administrator privileges If a minion that was configured with this script is already running, the script will exit. -The following example will set up a minion for the current logged in account. It +The following example sets up a minion for the current logged in account. It configures the minion to connect to the master at 192.168.0.10 .EXAMPLE PS>multi-minion.ps1 -Master 192.168.0.10 PS>multi-minion.ps1 -m 192.168.0.10 -The following example will set up a minion for the current logged in account. It -configures the minion to connect to the master at 192.168.0.10. It will also -prefix the minion id with `spongebob` +The following example sets up a minion for the current logged in account. It +configures the minion to connect to the master at 192.168.0.10. It also prefixes +the minion id with `spongebob` .EXAMPLE PS>multi-minion.ps1 -Master 192.168.0.10 -Prefix spongebob PS>multi-minion.ps1 -m 192.168.0.10 -p spongebob -The following example will set up a minion for the current logged in account. It -configures the minion to connect to the master at 192.168.0.10. It will also -start the minion in a hidden window: +The following example sets up a minion for the current logged in account. It +configures the minion to connect to the master at 192.168.0.10. It also starts +the minion in a hidden window: .EXAMPLE PS>multi-minion.ps1 -Master 192.168.0.10 -Start PS>multi-minion.ps1 -m 192.168.0.10 -s -The following example will remove a multiminion for the current running account: +The following example removes a multiminion for the current running account: .EXAMPLE PS>multi-minion.ps1 -Delete @@ -60,7 +60,7 @@ param( [Alias("p")] # The prefix to the minion id to differentiate it from the installed system # minion. The default is $env:COMPUTERNAME. It might be helpful to use the - # minion id of the System minion if you know it + # minion id of the system minion if you know it [String] $Prefix = "$env:COMPUTERNAME", [Parameter(Mandatory=$false)] @@ -82,7 +82,7 @@ param( "critical", "quiet" )] - # Start the minion in the background + # Set the log level for log file. Default is `warning` [String] $LogLevel = "warning", [Parameter(Mandatory=$false)] From eabc42cce72ce185fdbe36df9cd9e5f6b027fb78 Mon Sep 17 00:00:00 2001 From: jeanluc Date: Wed, 21 Jun 2023 13:40:20 +0200 Subject: [PATCH 14/53] Add tests for salt-ssh `state.*` exitcodes --- tests/pytests/integration/ssh/test_state.py | 341 ++++++++++++++++++++ 1 file changed, 341 insertions(+) diff --git a/tests/pytests/integration/ssh/test_state.py b/tests/pytests/integration/ssh/test_state.py index 9d3a38d2c9f..5f9bfb45e9f 100644 --- a/tests/pytests/integration/ssh/test_state.py +++ b/tests/pytests/integration/ssh/test_state.py @@ -2,6 +2,8 @@ import json import pytest +from salt.defaults.exitcodes import EX_AGGREGATE + pytestmark = [ pytest.mark.skip_on_windows(reason="salt-ssh not available on Windows"), ] @@ -75,6 +77,129 @@ def state_tree_dir(base_env_state_tree_root_dir): yield +@pytest.fixture(scope="class") +def state_tree_render_fail(base_env_state_tree_root_dir): + top_file = """ + base: + 'localhost': + - fail_render + '127.0.0.1': + - fail_render + """ + state_file = r""" + abc var is not defined {{ abc }}: + test.nop + """ + top_tempfile = pytest.helpers.temp_file( + "top.sls", top_file, base_env_state_tree_root_dir + ) + state_tempfile = pytest.helpers.temp_file( + "fail_render.sls", state_file, base_env_state_tree_root_dir + ) + with top_tempfile, state_tempfile: + yield + + +@pytest.fixture(scope="class") +def state_tree_req_fail(base_env_state_tree_root_dir): + top_file = """ + base: + 'localhost': + - fail_req + '127.0.0.1': + - fail_req + """ + state_file = """ + This has an invalid requisite: + test.nop: + - name: foo + - require_in: + - file.managed: invalid_requisite + """ + top_tempfile = pytest.helpers.temp_file( + "top.sls", top_file, base_env_state_tree_root_dir + ) + state_tempfile = pytest.helpers.temp_file( + "fail_req.sls", state_file, base_env_state_tree_root_dir + ) + with top_tempfile, state_tempfile: + yield + + +@pytest.fixture(scope="class") +def state_tree_structure_fail(base_env_state_tree_root_dir): + top_file = """ + base: + 'localhost': + - fail_structure + '127.0.0.1': + - fail_structure + """ + state_file = """ + extend: + Some file state: + file: + - name: /tmp/bar + - contents: bar + """ + top_tempfile = pytest.helpers.temp_file( + "top.sls", top_file, base_env_state_tree_root_dir + ) + state_tempfile = pytest.helpers.temp_file( + "fail_structure.sls", state_file, base_env_state_tree_root_dir + ) + with top_tempfile, state_tempfile: + yield + + +@pytest.fixture(scope="class") +def state_tree_run_fail(base_env_state_tree_root_dir): + top_file = """ + base: + 'localhost': + - fail_run + '127.0.0.1': + - fail_run + """ + state_file = """ + This file state fails: + file.managed: + - name: /tmp/non/ex/is/tent + - makedirs: false + - contents: foo + """ + top_tempfile = pytest.helpers.temp_file( + "top.sls", top_file, base_env_state_tree_root_dir + ) + state_tempfile = pytest.helpers.temp_file( + "fail_run.sls", state_file, base_env_state_tree_root_dir + ) + with top_tempfile, state_tempfile: + yield + + +@pytest.fixture(scope="class") +def pillar_tree_render_fail(base_env_pillar_tree_root_dir): + top_file = """ + base: + 'localhost': + - fail_render + '127.0.0.1': + - fail_render + """ + pillar_file = r""" + not_defined: {{ abc }} + """ + top_tempfile = pytest.helpers.temp_file( + "top.sls", top_file, base_env_pillar_tree_root_dir + ) + pillar_tempfile = pytest.helpers.temp_file( + "fail_render.sls", pillar_file, base_env_pillar_tree_root_dir + ) + with top_tempfile, pillar_tempfile: + yield + + @pytest.mark.slow_test def test_state_with_import(salt_ssh_cli, state_tree): """ @@ -220,3 +345,219 @@ def test_state_high(salt_ssh_cli): ]["stdout"] == "blah" ) + + +@pytest.mark.slow_test +@pytest.mark.usefixtures("state_tree_render_fail") +class TestRenderExceptionRetcode: + """ + Verify salt-ssh fails with a retcode > 0 when a state rendering fails. + """ + + def test_retcode_state_sls_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.sls", "fail_render") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_highstate_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.highstate") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_sls_id_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.sls_id", "foo", "fail_render") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_show_sls_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_sls", "fail_render") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_show_low_sls_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_low_sls", "fail_render") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_show_highstate_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_highstate") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_show_lowstate_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_lowstate") + # state.show_lowstate exits with 0 for non-ssh as well + self._assert_ret(ret, 0) + + def test_retcode_state_top_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.top", "top.sls") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_single_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.single", "file") + assert ret.returncode == EX_AGGREGATE + assert isinstance(ret.data, str) + assert "single() missing 1 required positional argument" in ret.data + + def _assert_ret(self, ret, retcode): + assert ret.returncode == retcode + assert isinstance(ret.data, list) + assert ret.data + assert isinstance(ret.data[0], str) + assert ret.data[0].startswith( + "Rendering SLS 'base:fail_render' failed: Jinja variable 'abc' is undefined;" + ) + + +@pytest.mark.slow_test +@pytest.mark.usefixtures("pillar_tree_render_fail") +class TestPillarRenderExceptionRetcode: + """ + Verify salt-ssh fails with a retcode > 0 when a pillar rendering fails. + """ + + def test_retcode_state_sls_pillar_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.sls", "basic") + self._assert_ret(ret) + + def test_retcode_state_highstate_pillar_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.highstate") + self._assert_ret(ret) + + def test_retcode_state_sls_id_pillar_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.sls_id", "foo", "basic") + self._assert_ret(ret) + + def test_retcode_state_show_sls_pillar_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_sls", "basic") + self._assert_ret(ret) + + def test_retcode_state_show_low_sls_pillar_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_low_sls", "basic") + self._assert_ret(ret) + + def test_retcode_state_show_highstate_pillar_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_highstate") + self._assert_ret(ret) + + def test_retcode_state_show_lowstate_pillar_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_lowstate") + self._assert_ret(ret) + + def test_retcode_state_top_pillar_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.top", "top.sls") + self._assert_ret(ret) + + def _assert_ret(self, ret): + assert ret.returncode == EX_AGGREGATE + assert isinstance(ret.data, list) + assert ret.data + assert isinstance(ret.data[0], str) + assert ret.data[0] == "Pillar failed to render with the following messages:" + assert ret.data[1].startswith("Rendering SLS 'fail_render' failed.") + + +@pytest.mark.slow_test +@pytest.mark.usefixtures("state_tree_req_fail") +class TestStateReqFailRetcode: + """ + Verify salt-ssh fails with a retcode > 0 when a highstate verification fails. + ``state.show_highstate`` does not validate this. + """ + + def test_retcode_state_sls_invalid_requisite(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.sls", "fail_req") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_highstate_invalid_requisite(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.highstate") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_show_sls_invalid_requisite(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_sls", "fail_req") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_show_low_sls_invalid_requisite(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_low_sls", "fail_req") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_show_lowstate_invalid_requisite(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_lowstate") + # state.show_lowstate exits with 0 for non-ssh as well + self._assert_ret(ret, 0) + + def test_retcode_state_top_invalid_requisite(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.top", "top.sls") + self._assert_ret(ret, EX_AGGREGATE) + + def _assert_ret(self, ret, retcode): + assert ret.returncode == retcode + assert isinstance(ret.data, list) + assert ret.data + assert isinstance(ret.data[0], str) + assert ret.data[0].startswith( + "Invalid requisite in require: file.managed for invalid_requisite" + ) + + +@pytest.mark.slow_test +@pytest.mark.usefixtures("state_tree_structure_fail") +class TestStateStructureFailRetcode: + """ + Verify salt-ssh fails with a retcode > 0 when a highstate verification fails. + This targets another step of the verification. + ``state.sls_id`` does not seem to support extends. + ``state.show_highstate`` does not validate this. + """ + + def test_retcode_state_sls_invalid_structure(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.sls", "fail_structure") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_highstate_invalid_structure(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.highstate") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_show_sls_invalid_structure(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_sls", "fail_structure") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_show_low_sls_invalid_structure(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_low_sls", "fail_structure") + self._assert_ret(ret, EX_AGGREGATE) + + def test_retcode_state_show_lowstate_invalid_structure(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.show_lowstate") + # state.show_lowstate exits with 0 for non-ssh as well + self._assert_ret(ret, 0) + + def test_retcode_state_top_invalid_structure(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.top", "top.sls") + self._assert_ret(ret, EX_AGGREGATE) + + def _assert_ret(self, ret, retcode): + assert ret.returncode == retcode + assert isinstance(ret.data, list) + assert ret.data + assert isinstance(ret.data[0], str) + assert ret.data[0].startswith( + "Cannot extend ID 'Some file state' in 'base:fail_structure" + ) + + +@pytest.mark.slow_test +@pytest.mark.usefixtures("state_tree_run_fail") +class TestStateRunFailRetcode: + """ + Verify salt-ssh passes on a failing retcode from state execution. + """ + + def test_retcode_state_sls_run_fail(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.sls", "fail_run") + assert ret.returncode == EX_AGGREGATE + + def test_retcode_state_highstate_run_fail(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.highstate") + assert ret.returncode == EX_AGGREGATE + + def test_retcode_state_sls_id_render_exception(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.sls_id", "This file state fails", "fail_run") + assert ret.returncode == EX_AGGREGATE + + def test_retcode_state_top_run_fail(self, salt_ssh_cli): + ret = salt_ssh_cli.run("state.top", "top.sls") + assert ret.returncode == EX_AGGREGATE From b58190a56e1986dfca357472c113afb26ce5535a Mon Sep 17 00:00:00 2001 From: jeanluc Date: Wed, 21 Jun 2023 15:38:32 +0200 Subject: [PATCH 15/53] Fix salt-ssh state.* commands retcode for render fail --- changelog/64514.fixed.md | 1 + salt/client/ssh/__init__.py | 40 ++++++++++-------- salt/client/ssh/wrapper/state.py | 70 ++++++++++++++++++++++++++++---- 3 files changed, 84 insertions(+), 27 deletions(-) create mode 100644 changelog/64514.fixed.md diff --git a/changelog/64514.fixed.md b/changelog/64514.fixed.md new file mode 100644 index 00000000000..b84fb366bf1 --- /dev/null +++ b/changelog/64514.fixed.md @@ -0,0 +1 @@ +Fixed salt-ssh state.* commands returning retcode 0 when state/pillar rendering fails diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index 88365a60994..4a1e785e6b6 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -552,6 +552,11 @@ class SSH(MultiprocessingStateMixin): data = salt.utils.json.find_json(stdout) if len(data) < 2 and "local" in data: ret["ret"] = data["local"] + try: + # Ensure a reported local retcode is kept + retcode = data["local"]["retcode"] + except (KeyError, TypeError): + pass else: ret["ret"] = { "stdout": stdout, @@ -564,7 +569,7 @@ class SSH(MultiprocessingStateMixin): "stderr": stderr, "retcode": retcode, } - que.put(ret) + que.put((ret, retcode)) def handle_ssh(self, mine=False): """ @@ -608,7 +613,7 @@ class SSH(MultiprocessingStateMixin): "fun": "", "id": host, } - yield {host: no_ret} + yield {host: no_ret}, 1 continue args = ( que, @@ -622,11 +627,12 @@ class SSH(MultiprocessingStateMixin): running[host] = {"thread": routine} continue ret = {} + retcode = 0 try: - ret = que.get(False) + ret, retcode = que.get(False) if "id" in ret: returned.add(ret["id"]) - yield {ret["id"]: ret["ret"]} + yield {ret["id"]: ret["ret"]}, retcode except queue.Empty: pass for host in running: @@ -636,10 +642,10 @@ class SSH(MultiprocessingStateMixin): # last checked try: while True: - ret = que.get(False) + ret, retcode = que.get(False) if "id" in ret: returned.add(ret["id"]) - yield {ret["id"]: ret["ret"]} + yield {ret["id"]: ret["ret"]}, retcode except queue.Empty: pass @@ -650,7 +656,7 @@ class SSH(MultiprocessingStateMixin): ) ret = {"id": host, "ret": error} log.error(error) - yield {ret["id"]: ret["ret"]} + yield {ret["id"]: ret["ret"]}, 1 running[host]["thread"].join() rets.add(host) for host in rets: @@ -705,8 +711,8 @@ class SSH(MultiprocessingStateMixin): jid, job_load ) - for ret in self.handle_ssh(mine=mine): - host = next(iter(ret.keys())) + for ret, _ in self.handle_ssh(mine=mine): + host = next(iter(ret)) self.cache_job(jid, host, ret[host], fun) if self.event: id_, data = next(iter(ret.items())) @@ -799,15 +805,9 @@ class SSH(MultiprocessingStateMixin): sret = {} outputter = self.opts.get("output", "nested") final_exit = 0 - for ret in self.handle_ssh(): - host = next(iter(ret.keys())) - if isinstance(ret[host], dict): - host_ret = ret[host].get("retcode", 0) - if host_ret != 0: - final_exit = 1 - else: - # Error on host - final_exit = 1 + for ret, retcode in self.handle_ssh(): + host = next(iter(ret)) + final_exit = max(final_exit, retcode) self.cache_job(jid, host, ret[host], fun) ret = self.key_deploy(host, ret) @@ -1274,6 +1274,10 @@ class Single: ) log.error(result, exc_info_on_loglevel=logging.DEBUG) retcode = 1 + + # Ensure retcode from wrappers is respected, especially state render exceptions + retcode = max(retcode, self.context.get("retcode", 0)) + # Mimic the json data-structure that "salt-call --local" will # emit (as seen in ssh_py_shim.py) if isinstance(result, dict) and "local" in result: diff --git a/salt/client/ssh/wrapper/state.py b/salt/client/ssh/wrapper/state.py index 002853972ab..0a1d5bdf5f9 100644 --- a/salt/client/ssh/wrapper/state.py +++ b/salt/client/ssh/wrapper/state.py @@ -8,6 +8,7 @@ import time import salt.client.ssh.shell import salt.client.ssh.state +import salt.defaults.exitcodes import salt.loader import salt.minion import salt.roster @@ -84,14 +85,14 @@ def _set_retcode(ret, highstate=None): """ # Set default retcode to 0 - __context__["retcode"] = 0 + __context__["retcode"] = salt.defaults.exitcodes.EX_OK if isinstance(ret, list): - __context__["retcode"] = 1 + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return if not salt.utils.state.check_result(ret, highstate=highstate): - __context__["retcode"] = 2 + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_FAILURE def _check_pillar(kwargs, pillar=None): @@ -182,6 +183,11 @@ def sls(mods, saltenv="base", test=None, exclude=None, **kwargs): __context__["fileclient"], context=__context__.value(), ) as st_: + if not _check_pillar(kwargs, st_.opts["pillar"]): + __context__["retcode"] = salt.defaults.exitcodes.EX_PILLAR_FAILURE + err = ["Pillar failed to render with the following messages:"] + err += st_.opts["pillar"]["_errors"] + return err st_.push_active() mods = _parse_mods(mods) high_data, errors = st_.render_highstate( @@ -198,12 +204,14 @@ def sls(mods, saltenv="base", test=None, exclude=None, **kwargs): errors += ext_errors errors += st_.state.verify_high(high_data) if errors: + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return errors high_data, req_in_errors = st_.state.requisite_in(high_data) errors += req_in_errors high_data = st_.state.apply_exclude(high_data) # Verify that the high data is structurally sound if errors: + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return errors # Compile and verify the raw chunks chunks = st_.state.compile_high_data(high_data) @@ -316,7 +324,7 @@ def _check_queue(queue, kwargs): else: conflict = running(concurrent=kwargs.get("concurrent", False)) if conflict: - __context__["retcode"] = 1 + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return conflict @@ -681,6 +689,11 @@ def highstate(test=None, **kwargs): __context__["fileclient"], context=__context__.value(), ) as st_: + if not _check_pillar(kwargs, st_.opts["pillar"]): + __context__["retcode"] = salt.defaults.exitcodes.EX_PILLAR_FAILURE + err = ["Pillar failed to render with the following messages:"] + err += st_.opts["pillar"]["_errors"] + return err st_.push_active() chunks = st_.compile_low_chunks(context=__context__.value()) file_refs = salt.client.ssh.state.lowstate_file_refs( @@ -692,7 +705,7 @@ def highstate(test=None, **kwargs): # Check for errors for chunk in chunks: if not isinstance(chunk, dict): - __context__["retcode"] = 1 + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return chunks roster = salt.roster.Roster(opts, opts.get("roster", "flat")) @@ -766,9 +779,19 @@ def top(topfn, test=None, **kwargs): __context__["fileclient"], context=__context__.value(), ) as st_: + if not _check_pillar(kwargs, st_.opts["pillar"]): + __context__["retcode"] = salt.defaults.exitcodes.EX_PILLAR_FAILURE + err = ["Pillar failed to render with the following messages:"] + err += st_.opts["pillar"]["_errors"] + return err st_.opts["state_top"] = os.path.join("salt://", topfn) st_.push_active() chunks = st_.compile_low_chunks(context=__context__.value()) + # Check for errors + for chunk in chunks: + if not isinstance(chunk, dict): + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR + return chunks file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs( @@ -839,8 +862,17 @@ def show_highstate(**kwargs): __context__["fileclient"], context=__context__.value(), ) as st_: + if not _check_pillar(kwargs, st_.opts["pillar"]): + __context__["retcode"] = salt.defaults.exitcodes.EX_PILLAR_FAILURE + err = ["Pillar failed to render with the following messages:"] + err += st_.opts["pillar"]["_errors"] + return err st_.push_active() chunks = st_.compile_highstate(context=__context__.value()) + # Check for errors + if not isinstance(chunks, dict): + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR + return chunks _cleanup_slsmod_high_data(chunks) return chunks @@ -864,6 +896,11 @@ def show_lowstate(**kwargs): __context__["fileclient"], context=__context__.value(), ) as st_: + if not _check_pillar(kwargs, st_.opts["pillar"]): + __context__["retcode"] = salt.defaults.exitcodes.EX_PILLAR_FAILURE + err = ["Pillar failed to render with the following messages:"] + err += st_.opts["pillar"]["_errors"] + return err st_.push_active() chunks = st_.compile_low_chunks(context=__context__.value()) _cleanup_slsmod_low_data(chunks) @@ -925,7 +962,7 @@ def sls_id(id_, mods, test=None, queue=False, **kwargs): ) as st_: if not _check_pillar(kwargs, st_.opts["pillar"]): - __context__["retcode"] = 5 + __context__["retcode"] = salt.defaults.exitcodes.EX_PILLAR_FAILURE err = ["Pillar failed to render with the following messages:"] err += __pillar__["_errors"] return err @@ -943,7 +980,7 @@ def sls_id(id_, mods, test=None, queue=False, **kwargs): # but it is required to get the unit tests to pass. errors.extend(req_in_errors) if errors: - __context__["retcode"] = 1 + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return errors chunks = st_.state.compile_high_data(high_) chunk = [x for x in chunks if x.get("__id__", "") == id_] @@ -988,6 +1025,11 @@ def show_sls(mods, saltenv="base", test=None, **kwargs): __context__["fileclient"], context=__context__.value(), ) as st_: + if not _check_pillar(kwargs, st_.opts["pillar"]): + __context__["retcode"] = salt.defaults.exitcodes.EX_PILLAR_FAILURE + err = ["Pillar failed to render with the following messages:"] + err += st_.opts["pillar"]["_errors"] + return err st_.push_active() mods = _parse_mods(mods) high_data, errors = st_.render_highstate( @@ -997,12 +1039,14 @@ def show_sls(mods, saltenv="base", test=None, **kwargs): errors += ext_errors errors += st_.state.verify_high(high_data) if errors: + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return errors high_data, req_in_errors = st_.state.requisite_in(high_data) errors += req_in_errors high_data = st_.state.apply_exclude(high_data) # Verify that the high data is structurally sound if errors: + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return errors _cleanup_slsmod_high_data(high_data) return high_data @@ -1036,6 +1080,11 @@ def show_low_sls(mods, saltenv="base", test=None, **kwargs): __context__["fileclient"], context=__context__.value(), ) as st_: + if not _check_pillar(kwargs, st_.opts["pillar"]): + __context__["retcode"] = salt.defaults.exitcodes.EX_PILLAR_FAILURE + err = ["Pillar failed to render with the following messages:"] + err += st_.opts["pillar"]["_errors"] + return err st_.push_active() mods = _parse_mods(mods) high_data, errors = st_.render_highstate( @@ -1045,12 +1094,14 @@ def show_low_sls(mods, saltenv="base", test=None, **kwargs): errors += ext_errors errors += st_.state.verify_high(high_data) if errors: + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return errors high_data, req_in_errors = st_.state.requisite_in(high_data) errors += req_in_errors high_data = st_.state.apply_exclude(high_data) # Verify that the high data is structurally sound if errors: + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return errors ret = st_.state.compile_high_data(high_data) _cleanup_slsmod_low_data(ret) @@ -1080,6 +1131,7 @@ def show_top(**kwargs): errors = [] errors += st_.verify_tops(top_data) if errors: + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return errors matches = st_.top_matches(top_data) return matches @@ -1110,7 +1162,7 @@ def single(fun, name, test=None, **kwargs): # state.fun -> [state, fun] comps = fun.split(".") if len(comps) < 2: - __context__["retcode"] = 1 + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return "Invalid function passed" # Create the low chunk, using kwargs as a base @@ -1133,7 +1185,7 @@ def single(fun, name, test=None, **kwargs): # Verify the low chunk err = st_.verify_data(kwargs) if err: - __context__["retcode"] = 1 + __context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR return err # Must be a list of low-chunks From a86675d4139b857de41e6b6c44b43a9f24a95936 Mon Sep 17 00:00:00 2001 From: jeanluc Date: Wed, 21 Jun 2023 15:40:54 +0200 Subject: [PATCH 16/53] Add misc tests for retcode passthrough --- tests/pytests/integration/ssh/test_deploy.py | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/pytests/integration/ssh/test_deploy.py b/tests/pytests/integration/ssh/test_deploy.py index dac28f84f37..d70f8fce23b 100644 --- a/tests/pytests/integration/ssh/test_deploy.py +++ b/tests/pytests/integration/ssh/test_deploy.py @@ -9,6 +9,7 @@ import pytest import salt.utils.files import salt.utils.yaml +from salt.defaults.exitcodes import EX_AGGREGATE pytestmark = [ pytest.mark.slow_test, @@ -100,3 +101,26 @@ def test_tty(salt_ssh_cli, tmp_path, salt_ssh_roster_file): ret = salt_ssh_cli.run("--roster-file={}".format(roster_file), "test.ping") assert ret.returncode == 0 assert ret.data is True + + +def test_retcode_exe_run_fail(salt_ssh_cli): + """ + Verify salt-ssh passes through the retcode it receives. + """ + ret = salt_ssh_cli.run("file.touch", "/tmp/non/ex/is/tent") + assert ret.returncode == EX_AGGREGATE + assert isinstance(ret.data, dict) + assert ret.data["stderr"] == "Error running 'file.touch': No such file or directory" + assert ret.data["retcode"] == 1 + + +def test_retcode_exe_run_exception(salt_ssh_cli): + """ + Verify salt-ssh passes through the retcode it receives + when an exception is thrown. (Ref #50727) + """ + ret = salt_ssh_cli.run("salttest.jinja_error") + assert ret.returncode == EX_AGGREGATE + assert isinstance(ret.data, dict) + assert ret.data["stderr"].endswith("Exception: hehehe") + assert ret.data["retcode"] == 1 From 9f70585e344be51fd774ceb7a5ee7b81c91822a4 Mon Sep 17 00:00:00 2001 From: jeanluc Date: Wed, 21 Jun 2023 15:41:52 +0200 Subject: [PATCH 17/53] run pre-commit upgrade code for Py3.8+ --- salt/client/ssh/__init__.py | 50 +++++++++----------- salt/client/ssh/wrapper/state.py | 18 +++---- tests/pytests/integration/ssh/test_deploy.py | 8 ++-- 3 files changed, 34 insertions(+), 42 deletions(-) diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index 4a1e785e6b6..0d9ac9509b6 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -304,7 +304,7 @@ class SSH(MultiprocessingStateMixin): } if self.opts.get("rand_thin_dir"): self.defaults["thin_dir"] = os.path.join( - "/var/tmp", ".{}".format(uuid.uuid4().hex[:6]) + "/var/tmp", f".{uuid.uuid4().hex[:6]}" ) self.opts["ssh_wipe"] = "True" self.returners = salt.loader.returners(self.opts, {}) @@ -454,9 +454,9 @@ class SSH(MultiprocessingStateMixin): priv = self.opts.get( "ssh_priv", os.path.join(self.opts["pki_dir"], "ssh", "salt-ssh.rsa") ) - pub = "{}.pub".format(priv) + pub = f"{priv}.pub" with salt.utils.files.fopen(pub, "r") as fp_: - return "{} rsa root@master".format(fp_.read().split()[1]) + return f"{fp_.read().split()[1]} rsa root@master" def key_deploy(self, host, ret): """ @@ -500,7 +500,7 @@ class SSH(MultiprocessingStateMixin): mods=self.mods, fsclient=self.fsclient, thin=self.thin, - **target + **target, ) if salt.utils.path.which("ssh-copy-id"): # we have ssh-copy-id, use it! @@ -516,7 +516,7 @@ class SSH(MultiprocessingStateMixin): mods=self.mods, fsclient=self.fsclient, thin=self.thin, - **target + **target, ) stdout, stderr, retcode = single.cmd_block() try: @@ -543,7 +543,7 @@ class SSH(MultiprocessingStateMixin): fsclient=self.fsclient, thin=self.thin, mine=mine, - **target + **target, ) ret = {"id": single.id} stdout, stderr, retcode = single.run() @@ -798,7 +798,7 @@ class SSH(MultiprocessingStateMixin): ) if self.opts.get("verbose"): - msg = "Executing job with jid {}".format(jid) + msg = f"Executing job with jid {jid}" print(msg) print("-" * len(msg) + "\n") print("") @@ -883,7 +883,7 @@ class Single: remote_port_forwards=None, winrm=False, ssh_options=None, - **kwargs + **kwargs, ): # Get mine setting and mine_functions if defined in kwargs (from roster) self.mine = mine @@ -1017,9 +1017,7 @@ class Single: """ check if the thindir exists on the remote machine """ - stdout, stderr, retcode = self.shell.exec_cmd( - "test -d {}".format(self.thin_dir) - ) + stdout, stderr, retcode = self.shell.exec_cmd(f"test -d {self.thin_dir}") if retcode != 0: return False return True @@ -1131,7 +1129,7 @@ class Single: self.id, fsclient=self.fsclient, minion_opts=self.minion_opts, - **self.target + **self.target, ) opts_pkg = pre_wrapper["test.opts_pkg"]() # pylint: disable=E1102 @@ -1217,7 +1215,7 @@ class Single: self.id, fsclient=self.fsclient, minion_opts=self.minion_opts, - **self.target + **self.target, ) wrapper.fsclient.opts["cachedir"] = opts["cachedir"] self.wfuncs = salt.loader.ssh_wrapper(opts, wrapper, self.context) @@ -1265,7 +1263,7 @@ class Single: else: result = self.wfuncs[self.fun](*self.args, **self.kwargs) except TypeError as exc: - result = "TypeError encountered executing {}: {}".format(self.fun, exc) + result = f"TypeError encountered executing {self.fun}: {exc}" log.error(result, exc_info_on_loglevel=logging.DEBUG) retcode = 1 except Exception as exc: # pylint: disable=broad-except @@ -1292,7 +1290,7 @@ class Single: """ if self.target.get("sudo"): sudo = ( - "sudo -p '{}'".format(salt.client.ssh.shell.SUDO_PROMPT) + f"sudo -p '{salt.client.ssh.shell.SUDO_PROMPT}'" if self.target.get("passwd") else "sudo" ) @@ -1364,20 +1362,18 @@ ARGS = {arguments}\n'''.format( script_args = shlex.split(str(script_args)) args = " {}".format(" ".join([shlex.quote(str(el)) for el in script_args])) if extension == "ps1": - ret = self.shell.exec_cmd('"powershell {}"'.format(script)) + ret = self.shell.exec_cmd(f'"powershell {script}"') else: if not self.winrm: - ret = self.shell.exec_cmd( - "/bin/sh '{}{}'{}".format(pre_dir, script, args) - ) + ret = self.shell.exec_cmd(f"/bin/sh '{pre_dir}{script}'{args}") else: ret = saltwinshell.call_python(self, script) # Remove file from target system if not self.winrm: - self.shell.exec_cmd("rm '{}{}'".format(pre_dir, script)) + self.shell.exec_cmd(f"rm '{pre_dir}{script}'") else: - self.shell.exec_cmd("del {}".format(script)) + self.shell.exec_cmd(f"del {script}") return ret @@ -1465,7 +1461,7 @@ ARGS = {arguments}\n'''.format( while re.search(RSTR_RE, stderr): stderr = re.split(RSTR_RE, stderr, 1)[1].strip() else: - return "ERROR: {}".format(error), stderr, retcode + return f"ERROR: {error}", stderr, retcode # FIXME: this discards output from ssh_shim if the shim succeeds. It should # always save the shim output regardless of shim success or failure. @@ -1525,7 +1521,7 @@ ARGS = {arguments}\n'''.format( # If RSTR is not seen in both stdout and stderr then there # was a thin deployment problem. return ( - "ERROR: Failure deploying ext_mods: {}".format(stdout), + f"ERROR: Failure deploying ext_mods: {stdout}", stderr, retcode, ) @@ -1693,7 +1689,7 @@ def mod_data(fsclient): files = fsclient.file_list(env) for ref in sync_refs: mods_data = {} - pref = "_{}".format(ref) + pref = f"_{ref}" for fn_ in sorted(files): if fn_.startswith(pref): if fn_.endswith((".py", ".so", ".pyx")): @@ -1715,9 +1711,7 @@ def mod_data(fsclient): ver_base = salt.utils.stringutils.to_bytes(ver_base) ver = hashlib.sha1(ver_base).hexdigest() - ext_tar_path = os.path.join( - fsclient.opts["cachedir"], "ext_mods.{}.tgz".format(ver) - ) + ext_tar_path = os.path.join(fsclient.opts["cachedir"], f"ext_mods.{ver}.tgz") mods = {"version": ver, "file": ext_tar_path} if os.path.isfile(ext_tar_path): return mods @@ -1766,7 +1760,7 @@ def _convert_args(args): for key in list(arg.keys()): if key == "__kwarg__": continue - converted.append("{}={}".format(key, arg[key])) + converted.append(f"{key}={arg[key]}") else: converted.append(arg) return converted diff --git a/salt/client/ssh/wrapper/state.py b/salt/client/ssh/wrapper/state.py index 0a1d5bdf5f9..353d8a0e03e 100644 --- a/salt/client/ssh/wrapper/state.py +++ b/salt/client/ssh/wrapper/state.py @@ -55,7 +55,7 @@ def _ssh_state(chunks, st_kwargs, kwargs, test=False): cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, - **st_kwargs + **st_kwargs, ) single.shell.send(trans_tar, "{}/salt_state.tgz".format(__opts__["thin_dir"])) stdout, stderr, _ = single.cmd_block() @@ -244,7 +244,7 @@ def sls(mods, saltenv="base", test=None, exclude=None, **kwargs): cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, - **st_kwargs + **st_kwargs, ) single.shell.send(trans_tar, "{}/salt_state.tgz".format(opts["thin_dir"])) stdout, stderr, _ = single.cmd_block() @@ -392,7 +392,7 @@ def low(data, **kwargs): cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, - **st_kwargs + **st_kwargs, ) single.shell.send(trans_tar, "{}/salt_state.tgz".format(__opts__["thin_dir"])) stdout, stderr, _ = single.cmd_block() @@ -482,7 +482,7 @@ def high(data, **kwargs): cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, - **st_kwargs + **st_kwargs, ) single.shell.send(trans_tar, "{}/salt_state.tgz".format(opts["thin_dir"])) stdout, stderr, _ = single.cmd_block() @@ -558,7 +558,7 @@ def request(mods=None, **kwargs): try: if salt.utils.platform.is_windows(): # Make sure cache file isn't read-only - __salt__["cmd.run"]('attrib -R "{}"'.format(notify_path)) + __salt__["cmd.run"](f'attrib -R "{notify_path}"') with salt.utils.files.fopen(notify_path, "w+b") as fp_: salt.payload.dump(req, fp_) except OSError: @@ -622,7 +622,7 @@ def clear_request(name=None): try: if salt.utils.platform.is_windows(): # Make sure cache file isn't read-only - __salt__["cmd.run"]('attrib -R "{}"'.format(notify_path)) + __salt__["cmd.run"](f'attrib -R "{notify_path}"') with salt.utils.files.fopen(notify_path, "w+b") as fp_: salt.payload.dump(req, fp_) except OSError: @@ -730,7 +730,7 @@ def highstate(test=None, **kwargs): cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, - **st_kwargs + **st_kwargs, ) single.shell.send(trans_tar, "{}/salt_state.tgz".format(opts["thin_dir"])) stdout, stderr, _ = single.cmd_block() @@ -821,7 +821,7 @@ def top(topfn, test=None, **kwargs): cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, - **st_kwargs + **st_kwargs, ) single.shell.send(trans_tar, "{}/salt_state.tgz".format(opts["thin_dir"])) stdout, stderr, _ = single.cmd_block() @@ -1227,7 +1227,7 @@ def single(fun, name, test=None, **kwargs): cmd, fsclient=__context__["fileclient"], minion_opts=__salt__.minion_opts, - **st_kwargs + **st_kwargs, ) # Copy the tar down diff --git a/tests/pytests/integration/ssh/test_deploy.py b/tests/pytests/integration/ssh/test_deploy.py index d70f8fce23b..8512813f6e6 100644 --- a/tests/pytests/integration/ssh/test_deploy.py +++ b/tests/pytests/integration/ssh/test_deploy.py @@ -75,15 +75,13 @@ def test_set_path(salt_ssh_cli, tmp_path, salt_ssh_roster_file): roster_data = salt.utils.yaml.safe_load(rfh) roster_data["localhost"].update( { - "set_path": "$PATH:/usr/local/bin/:{}".format(path), + "set_path": f"$PATH:/usr/local/bin/:{path}", } ) with salt.utils.files.fopen(roster_file, "w") as wfh: salt.utils.yaml.safe_dump(roster_data, wfh) - ret = salt_ssh_cli.run( - "--roster-file={}".format(roster_file), "environ.get", "PATH" - ) + ret = salt_ssh_cli.run(f"--roster-file={roster_file}", "environ.get", "PATH") assert ret.returncode == 0 assert path in ret.data @@ -98,7 +96,7 @@ def test_tty(salt_ssh_cli, tmp_path, salt_ssh_roster_file): roster_data["localhost"].update({"tty": True}) with salt.utils.files.fopen(roster_file, "w") as wfh: salt.utils.yaml.safe_dump(roster_data, wfh) - ret = salt_ssh_cli.run("--roster-file={}".format(roster_file), "test.ping") + ret = salt_ssh_cli.run(f"--roster-file={roster_file}", "test.ping") assert ret.returncode == 0 assert ret.data is True From a7d7c4151bac4cabc195bc3fa2a04c2466fe91d2 Mon Sep 17 00:00:00 2001 From: MKLeb Date: Wed, 21 Jun 2023 17:58:17 -0400 Subject: [PATCH 18/53] Lock to `immutables>0.16`, as that version has problems installing now --- requirements/static/ci/py3.10/cloud.txt | 2 +- requirements/static/ci/py3.10/lint.txt | 2 +- requirements/static/ci/py3.7/cloud.txt | 2 +- requirements/static/ci/py3.7/lint.txt | 2 +- requirements/static/ci/py3.8/cloud.txt | 2 +- requirements/static/ci/py3.8/lint.txt | 2 +- requirements/static/ci/py3.9/cloud.txt | 2 +- requirements/static/ci/py3.9/lint.txt | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/requirements/static/ci/py3.10/cloud.txt b/requirements/static/ci/py3.10/cloud.txt index c43683e5e13..daf4d5e1570 100644 --- a/requirements/static/ci/py3.10/cloud.txt +++ b/requirements/static/ci/py3.10/cloud.txt @@ -441,7 +441,7 @@ idna==2.8 # etcd3-py # requests # yarl -immutables==0.16 +immutables==0.19 # via contextvars importlib-metadata==6.0.0 # via -r requirements/static/pkg/linux.in diff --git a/requirements/static/ci/py3.10/lint.txt b/requirements/static/ci/py3.10/lint.txt index 3ce019cab6f..72daefca405 100644 --- a/requirements/static/ci/py3.10/lint.txt +++ b/requirements/static/ci/py3.10/lint.txt @@ -438,7 +438,7 @@ idna==3.2 # etcd3-py # requests # yarl -immutables==0.16 +immutables==0.19 # via contextvars importlib-metadata==6.0.0 # via -r requirements/static/pkg/linux.in diff --git a/requirements/static/ci/py3.7/cloud.txt b/requirements/static/ci/py3.7/cloud.txt index 9fa1a9adff6..74e67f129c1 100644 --- a/requirements/static/ci/py3.7/cloud.txt +++ b/requirements/static/ci/py3.7/cloud.txt @@ -453,7 +453,7 @@ idna==2.8 # etcd3-py # requests # yarl -immutables==0.16 +immutables==0.19 # via contextvars importlib-metadata==4.8.1 # via diff --git a/requirements/static/ci/py3.7/lint.txt b/requirements/static/ci/py3.7/lint.txt index 1cabc19bb58..8d8771573c6 100644 --- a/requirements/static/ci/py3.7/lint.txt +++ b/requirements/static/ci/py3.7/lint.txt @@ -452,7 +452,7 @@ idna==3.2 # etcd3-py # requests # yarl -immutables==0.16 +immutables==0.19 # via contextvars importlib-metadata==4.6.4 # via diff --git a/requirements/static/ci/py3.8/cloud.txt b/requirements/static/ci/py3.8/cloud.txt index 0911e6619c0..fe4a7bcea2b 100644 --- a/requirements/static/ci/py3.8/cloud.txt +++ b/requirements/static/ci/py3.8/cloud.txt @@ -451,7 +451,7 @@ idna==2.8 # etcd3-py # requests # yarl -immutables==0.16 +immutables==0.19 # via contextvars importlib-metadata==4.8.1 # via -r requirements/static/pkg/linux.in diff --git a/requirements/static/ci/py3.8/lint.txt b/requirements/static/ci/py3.8/lint.txt index 3bd160808e6..b0a9aa31f4a 100644 --- a/requirements/static/ci/py3.8/lint.txt +++ b/requirements/static/ci/py3.8/lint.txt @@ -450,7 +450,7 @@ idna==3.2 # etcd3-py # requests # yarl -immutables==0.16 +immutables==0.19 # via contextvars importlib-metadata==4.6.4 # via -r requirements/static/pkg/linux.in diff --git a/requirements/static/ci/py3.9/cloud.txt b/requirements/static/ci/py3.9/cloud.txt index 9db7aaccdcb..ec7cb3da6ba 100644 --- a/requirements/static/ci/py3.9/cloud.txt +++ b/requirements/static/ci/py3.9/cloud.txt @@ -451,7 +451,7 @@ idna==2.8 # etcd3-py # requests # yarl -immutables==0.16 +immutables==0.19 # via contextvars importlib-metadata==6.0.0 # via -r requirements/static/pkg/linux.in diff --git a/requirements/static/ci/py3.9/lint.txt b/requirements/static/ci/py3.9/lint.txt index c7236117c66..50e4c72ed5e 100644 --- a/requirements/static/ci/py3.9/lint.txt +++ b/requirements/static/ci/py3.9/lint.txt @@ -448,7 +448,7 @@ idna==3.2 # etcd3-py # requests # yarl -immutables==0.16 +immutables==0.19 # via contextvars importlib-metadata==6.0.0 # via -r requirements/static/pkg/linux.in From c1292aecbf746a7704b1367901cdc7c934c109df Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Tue, 13 Jun 2023 19:33:09 +0000 Subject: [PATCH 19/53] add code to pass full_return through stack so exceptions can return retcode properly to salt-run cli --- changelog/61173.fixed.md | 1 + salt/client/mixins.py | 29 ++++++++++++++++--- salt/runner.py | 3 +- .../pytests/functional/cli/test_salt_run_.py | 10 +++++++ 4 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 changelog/61173.fixed.md create mode 100644 tests/pytests/functional/cli/test_salt_run_.py diff --git a/changelog/61173.fixed.md b/changelog/61173.fixed.md new file mode 100644 index 00000000000..edcb75b028f --- /dev/null +++ b/changelog/61173.fixed.md @@ -0,0 +1 @@ +fix fixed runner not having a proper exit code when runner modules throw an exception. diff --git a/salt/client/mixins.py b/salt/client/mixins.py index 7cdae88ae8a..27e12fea194 100644 --- a/salt/client/mixins.py +++ b/salt/client/mixins.py @@ -412,6 +412,7 @@ class SyncClientMixin(ClientStateMixin): traceback.format_exc(), ) data["success"] = False + data["retcode"] = 1 if self.store_job: try: @@ -480,7 +481,17 @@ class AsyncClientMixin(ClientStateMixin): @classmethod def _proc_function_remote( - cls, *, instance, opts, fun, low, user, tag, jid, daemonize=True + cls, + *, + instance, + opts, + fun, + low, + user, + tag, + jid, + daemonize=True, + full_return=False ): """ Run this method in a multiprocess target to execute the function on the @@ -506,13 +517,23 @@ class AsyncClientMixin(ClientStateMixin): instance = cls(opts) try: - return instance.cmd_sync(low) + return instance.cmd_sync(low, full_return=False) except salt.exceptions.EauthAuthenticationError as exc: log.error(exc) @classmethod def _proc_function( - cls, *, instance, opts, fun, low, user, tag, jid, daemonize=True + cls, + *, + instance, + opts, + fun, + low, + user, + tag, + jid, + daemonize=True, + full_return=False ): """ Run this method in a multiprocess target to execute the function @@ -537,7 +558,7 @@ class AsyncClientMixin(ClientStateMixin): low["__user__"] = user low["__tag__"] = tag - return instance.low(fun, low) + return instance.low(fun, low, full_return=full_return) def cmd_async(self, low): """ diff --git a/salt/runner.py b/salt/runner.py index caf6471fa08..d20cf275fe1 100644 --- a/salt/runner.py +++ b/salt/runner.py @@ -289,7 +289,7 @@ class Runner(RunnerClient): # otherwise run it in the main process if self.opts.get("eauth"): - ret = self.cmd_sync(low) + ret = self.cmd_sync(low, full_return=True) if isinstance(ret, dict) and set(ret) == {"data", "outputter"}: outputter = ret["outputter"] ret = ret["data"] @@ -306,6 +306,7 @@ class Runner(RunnerClient): tag=async_pub["tag"], jid=async_pub["jid"], daemonize=False, + full_return=True, ) except salt.exceptions.SaltException as exc: with salt.utils.event.get_event("master", opts=self.opts) as evt: diff --git a/tests/pytests/functional/cli/test_salt_run_.py b/tests/pytests/functional/cli/test_salt_run_.py new file mode 100644 index 00000000000..e41692894a6 --- /dev/null +++ b/tests/pytests/functional/cli/test_salt_run_.py @@ -0,0 +1,10 @@ +import logging + +log = logging.getLogger(__name__) + + +def test_exception_exit(salt_run_cli): + ret = salt_run_cli.run( + "error.error", "name='Exception'", "message='This is an error.'" + ) + assert ret.returncode == 1 From 0078f83664f06a479307c804bc4891b86069a116 Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Tue, 13 Jun 2023 20:05:08 +0000 Subject: [PATCH 20/53] add secondary tests for salt-run. they were not requested by ticket but should be tested to make sure they do not break --- .../pytests/functional/cli/test_salt_run_.py | 78 ++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/tests/pytests/functional/cli/test_salt_run_.py b/tests/pytests/functional/cli/test_salt_run_.py index e41692894a6..7f534f76987 100644 --- a/tests/pytests/functional/cli/test_salt_run_.py +++ b/tests/pytests/functional/cli/test_salt_run_.py @@ -1,10 +1,86 @@ import logging +import os + +import salt.version log = logging.getLogger(__name__) -def test_exception_exit(salt_run_cli): +def test_salt_run_exception_exit(salt_run_cli): + """ + test that the exitcode is 1 when an exception is + thrown in a salt runner + """ ret = salt_run_cli.run( "error.error", "name='Exception'", "message='This is an error.'" ) assert ret.returncode == 1 + + +def test_salt_run_non_exception_exit(salt_run_cli): + """ + Test standard exitcode and output when runner works. + """ + ret = salt_run_cli.run("test.stdout_print") + assert ret.returncode == 0 + assert ret.stdout == 'foo\n"bar"\n' + + +def test_versions_report(salt_run_cli): + """ + test salt-run --versions-report + """ + expected = salt.version.versions_information() + # sanitize expected of unnnecessary whitespace + for _, section in expected.items(): + for key in section: + if isinstance(section[key], str): + section[key] = section[key].strip() + + ret = salt_run_cli.run("--versions-report") + assert ret.returncode == 0 + assert ret.stdout + ret_lines = ret.stdout.split("\n") + + assert ret_lines + # sanitize lines + ret_lines = [line.strip() for line in ret_lines] + + for header in expected: + assert "{}:".format(header) in ret_lines + + ret_dict = {} + expected_keys = set() + for line in ret_lines: + if not line: + continue + if line.endswith(":"): + assert not expected_keys + current_header = line.rstrip(":") + assert current_header in expected + ret_dict[current_header] = {} + expected_keys = set(expected[current_header].keys()) + else: + key, *value_list = line.split(":", 1) + assert value_list + assert len(value_list) == 1 + value = value_list[0].strip() + if value == "Not Installed": + value = None + ret_dict[current_header][key] = value + assert key in expected_keys + expected_keys.remove(key) + assert not expected_keys + if os.environ.get("ONEDIR_TESTRUN", "0") == "0": + # Stop any more testing + return + + assert "relenv" in ret_dict["Dependency Versions"] + assert "Salt Extensions" in ret_dict + assert "salt-analytics-framework" in ret_dict["Salt Extensions"] + + +def test_salt_run_version(salt_run_cli): + expected = salt.version.__version__ + ret = salt_run_cli.run("--version") + assert f"cli_salt_run.py {expected}\n" == ret.stdout From 48ba8089e11c461ae6ba6cdc90dc1885d5bc7df8 Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Tue, 13 Jun 2023 20:11:07 +0000 Subject: [PATCH 21/53] fix full_return=False to follow flow of function, instead of force to False --- salt/client/mixins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/client/mixins.py b/salt/client/mixins.py index 27e12fea194..60861e73e7e 100644 --- a/salt/client/mixins.py +++ b/salt/client/mixins.py @@ -517,7 +517,7 @@ class AsyncClientMixin(ClientStateMixin): instance = cls(opts) try: - return instance.cmd_sync(low, full_return=False) + return instance.cmd_sync(low, full_return=full_return) except salt.exceptions.EauthAuthenticationError as exc: log.error(exc) From e9b1b14b97707b8c617b33713572546bac8e7414 Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Tue, 13 Jun 2023 21:52:51 +0000 Subject: [PATCH 22/53] try fixing broken eauth test --- salt/client/mixins.py | 14 ++------------ tests/pytests/functional/cli/test_salt_run_.py | 1 - 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/salt/client/mixins.py b/salt/client/mixins.py index 60861e73e7e..c6b479e106b 100644 --- a/salt/client/mixins.py +++ b/salt/client/mixins.py @@ -481,17 +481,7 @@ class AsyncClientMixin(ClientStateMixin): @classmethod def _proc_function_remote( - cls, - *, - instance, - opts, - fun, - low, - user, - tag, - jid, - daemonize=True, - full_return=False + cls, *, instance, opts, fun, low, user, tag, jid, daemonize=True ): """ Run this method in a multiprocess target to execute the function on the @@ -517,7 +507,7 @@ class AsyncClientMixin(ClientStateMixin): instance = cls(opts) try: - return instance.cmd_sync(low, full_return=full_return) + return instance.cmd_sync(low) except salt.exceptions.EauthAuthenticationError as exc: log.error(exc) diff --git a/tests/pytests/functional/cli/test_salt_run_.py b/tests/pytests/functional/cli/test_salt_run_.py index 7f534f76987..66c28fc3aae 100644 --- a/tests/pytests/functional/cli/test_salt_run_.py +++ b/tests/pytests/functional/cli/test_salt_run_.py @@ -77,7 +77,6 @@ def test_versions_report(salt_run_cli): assert "relenv" in ret_dict["Dependency Versions"] assert "Salt Extensions" in ret_dict - assert "salt-analytics-framework" in ret_dict["Salt Extensions"] def test_salt_run_version(salt_run_cli): From b662bbeb69e9e3a3ddc21101d8e717a84ab48cec Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Tue, 13 Jun 2023 22:05:19 +0000 Subject: [PATCH 23/53] no full_return for eauth --- salt/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/runner.py b/salt/runner.py index d20cf275fe1..d390af93cde 100644 --- a/salt/runner.py +++ b/salt/runner.py @@ -289,7 +289,7 @@ class Runner(RunnerClient): # otherwise run it in the main process if self.opts.get("eauth"): - ret = self.cmd_sync(low, full_return=True) + ret = self.cmd_sync(low) if isinstance(ret, dict) and set(ret) == {"data", "outputter"}: outputter = ret["outputter"] ret = ret["data"] From c608df741bef78511d449c82e081ddbfc56d7b60 Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Fri, 16 Jun 2023 18:54:07 +0000 Subject: [PATCH 24/53] fix slow_kill test by checking if return but no retcode in ret --- salt/cli/run.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/cli/run.py b/salt/cli/run.py index 93387479bca..c6f6f42c14e 100644 --- a/salt/cli/run.py +++ b/salt/cli/run.py @@ -38,6 +38,12 @@ class SaltRun(salt.utils.parsers.SaltRunOptionParser): # runners might still use it. For this reason, we # also check ret['data']['retcode'] if # ret['retcode'] is not available. + if ( + isinstance(ret, dict) + and "return" in ret + and "retcode" not in ret + ): + ret = ret["return"] if isinstance(ret, dict) and "retcode" in ret: self.exit(ret["retcode"]) elif isinstance(ret, dict) and "retcode" in ret.get("data", {}): From 79b5eeec967190c7fcfc9b906f30f4f14095323d Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Tue, 20 Jun 2023 12:34:50 -0700 Subject: [PATCH 25/53] Update changelog/61173.fixed.md remove fix Co-authored-by: Megan Wilhite --- changelog/61173.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/61173.fixed.md b/changelog/61173.fixed.md index edcb75b028f..d1e1161be7a 100644 --- a/changelog/61173.fixed.md +++ b/changelog/61173.fixed.md @@ -1 +1 @@ -fix fixed runner not having a proper exit code when runner modules throw an exception. +fixed runner not having a proper exit code when runner modules throw an exception. From 2f5c44a45eba58a8786e5050c5073f7d1171b830 Mon Sep 17 00:00:00 2001 From: MKLeb Date: Thu, 22 Jun 2023 11:49:45 -0400 Subject: [PATCH 26/53] Fixes for the `handle_ssh` return values now being lists of tuples --- .../pytests/unit/client/ssh/test_password.py | 19 +++++++++++-------- .../unit/client/ssh/test_return_events.py | 6 ++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/pytests/unit/client/ssh/test_password.py b/tests/pytests/unit/client/ssh/test_password.py index 8a7794d2f4a..71130691812 100644 --- a/tests/pytests/unit/client/ssh/test_password.py +++ b/tests/pytests/unit/client/ssh/test_password.py @@ -30,13 +30,16 @@ def test_password_failure(temp_salt_master, tmp_path): opts["arg"] = [] roster = str(tmp_path / "roster") handle_ssh_ret = [ - { - "localhost": { - "retcode": 255, - "stderr": "Permission denied (publickey).\r\n", - "stdout": "", - } - }, + ( + { + "localhost": { + "retcode": 255, + "stderr": "Permission denied (publickey).\r\n", + "stdout": "", + } + }, + 1, + ) ] expected = {"localhost": "Permission denied (publickey)"} display_output = MagicMock() @@ -50,4 +53,4 @@ def test_password_failure(temp_salt_master, tmp_path): with pytest.raises(SystemExit): client.run() display_output.assert_called_once_with(expected, "nested", opts) - assert ret is handle_ssh_ret[0] + assert ret is handle_ssh_ret[0][0] diff --git a/tests/pytests/unit/client/ssh/test_return_events.py b/tests/pytests/unit/client/ssh/test_return_events.py index 1f0b0dbf335..382a7b2e57a 100644 --- a/tests/pytests/unit/client/ssh/test_return_events.py +++ b/tests/pytests/unit/client/ssh/test_return_events.py @@ -26,9 +26,7 @@ def test_not_missing_fun_calling_wfuncs(temp_salt_master, tmp_path): opts["tgt"] = "localhost" opts["arg"] = [] roster = str(tmp_path / "roster") - handle_ssh_ret = [ - {"localhost": {}}, - ] + handle_ssh_ret = [({"localhost": {}}, 0)] expected = {"localhost": {}} display_output = MagicMock() @@ -44,7 +42,7 @@ def test_not_missing_fun_calling_wfuncs(temp_salt_master, tmp_path): assert "fun" in ret["localhost"] client.run() display_output.assert_called_once_with(expected, "nested", opts) - assert ret is handle_ssh_ret[0] + assert ret is handle_ssh_ret[0][0] assert len(client.event.fire_event.call_args_list) == 2 assert "fun" in client.event.fire_event.call_args_list[0][0][0] assert "fun" in client.event.fire_event.call_args_list[1][0][0] From dedade07d7e94ac31207cc473fe72de51f0fd234 Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Fri, 23 Jun 2023 17:32:00 +0000 Subject: [PATCH 27/53] fix nightlys by mimicing what cli.run does to runner.run output in run_run_plus --- tests/support/case.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/support/case.py b/tests/support/case.py index 55b537f6ffd..4b3222877b1 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -191,6 +191,12 @@ class ShellCase(TestCase, AdaptedConfigurationTestCaseMixin, ScriptPathMixin): with RedirectStdStreams(): runner = salt.runner.Runner(opts) ret["return"] = runner.run() + if ( + isinstance(ret["return"], dict) + and "return" in ret["return"] + and "retcode" not in ret["return"] + ): + ret["return"] = ret["return"]["return"] try: ret["jid"] = runner.jid except AttributeError: From 848b9dcfdd5823f91f413d739e6dfe3929529544 Mon Sep 17 00:00:00 2001 From: Megan Wilhite Date: Thu, 22 Jun 2023 11:20:04 -0600 Subject: [PATCH 28/53] Fix user.present state when group is unset --- changelog/64211.fixed.md | 1 + salt/states/user.py | 2 +- tests/pytests/functional/states/test_user.py | 72 ++++++++++++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 changelog/64211.fixed.md diff --git a/changelog/64211.fixed.md b/changelog/64211.fixed.md new file mode 100644 index 00000000000..26b39acf028 --- /dev/null +++ b/changelog/64211.fixed.md @@ -0,0 +1 @@ +Fix user.present state when groups is unset to ensure the groups are unchanged, as documented. diff --git a/salt/states/user.py b/salt/states/user.py index e0f620cac89..d575655cf34 100644 --- a/salt/states/user.py +++ b/salt/states/user.py @@ -127,7 +127,7 @@ def _changes( if _gid not in dupe_groups: dupe_groups[_gid] = [] dupe_groups[_gid].append(lusr["groups"][idx]) - if not remove_groups: + if not remove_groups or groups is None and not optional_groups: wanted_groups = sorted(set(wanted_groups + lusr["groups"])) if uid and lusr["uid"] != uid: change["uid"] = uid diff --git a/tests/pytests/functional/states/test_user.py b/tests/pytests/functional/states/test_user.py index 09d34da168a..7b334b51793 100644 --- a/tests/pytests/functional/states/test_user.py +++ b/tests/pytests/functional/states/test_user.py @@ -429,3 +429,75 @@ def test_user_present_change_optional_groups( user_info = modules.user.info(username) assert user_info assert user_info["groups"] == [group_1.name] + + +@pytest.mark.skip_unless_on_linux(reason="underlying functionality only runs on Linux") +def test_user_present_no_groups(modules, states, username): + """ + test user.present when groups arg is not + included by the group is created in another + state. Re-run the states to ensure there are + not changes and it is idempotent. + """ + groups = ["testgroup1", "testgroup2"] + try: + ret = states.group.present(name=username, gid=61121) + assert ret.result is True + + ret = states.user.present( + name=username, + uid=61121, + gid=61121, + ) + assert ret.result is True + assert ret.changes["groups"] == [username] + assert ret.changes["name"] == username + + ret = states.group.present( + name=groups[0], + members=[username], + ) + assert ret.changes["members"] == [username] + + ret = states.group.present( + name=groups[1], + members=[username], + ) + assert ret.changes["members"] == [username] + + user_info = modules.user.info(username) + assert user_info + assert user_info["groups"] == [username, groups[0], groups[1]] + + # run again, expecting no changes + ret = states.group.present(name=username) + assert ret.result is True + assert ret.changes == {} + + ret = states.user.present( + name=username, + ) + assert ret.result is True + assert ret.changes == {} + + ret = states.group.present( + name=groups[0], + members=[username], + ) + assert ret.result is True + assert ret.changes == {} + + ret = states.group.present( + name=groups[1], + members=[username], + ) + assert ret.result is True + assert ret.changes == {} + + user_info = modules.user.info(username) + assert user_info + assert user_info["groups"] == [username, groups[0], groups[1]] + finally: + for group in groups: + ret = states.group.absent(name=group) + assert ret.result is True From 617f9e92de5872f2ce6300f2dded83c4ffb8b5d2 Mon Sep 17 00:00:00 2001 From: Megan Wilhite Date: Fri, 23 Jun 2023 11:45:55 -0600 Subject: [PATCH 29/53] Fix user unit test --- tests/pytests/functional/states/test_user.py | 2 -- tests/pytests/unit/states/test_user.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytests/functional/states/test_user.py b/tests/pytests/functional/states/test_user.py index 7b334b51793..96b1ec55c88 100644 --- a/tests/pytests/functional/states/test_user.py +++ b/tests/pytests/functional/states/test_user.py @@ -117,7 +117,6 @@ def test_user_present_when_home_dir_does_not_18843(states, existing_account): ret = states.user.present( name=existing_account.username, home=existing_account.info.home, - remove_groups=False, ) assert ret.result is True assert pathlib.Path(existing_account.info.home).is_dir() @@ -228,7 +227,6 @@ def test_user_present_unicode(states, username, subtests): roomnumber="①②③", workphone="١٢٣٤", homephone="६७८", - remove_groups=False, ) assert ret.result is True diff --git a/tests/pytests/unit/states/test_user.py b/tests/pytests/unit/states/test_user.py index 0476cee40f8..ffbd2d7d4d1 100644 --- a/tests/pytests/unit/states/test_user.py +++ b/tests/pytests/unit/states/test_user.py @@ -189,6 +189,8 @@ def test_present_uid_gid_change(): "user.chgid": Mock(), "file.group_to_gid": mock_group_to_gid, "file.gid_to_group": mock_gid_to_group, + "group.info": MagicMock(return_value=after), + "user.chgroups": MagicMock(return_value=True), } with patch.dict(user.__grains__, {"kernel": "Linux"}), patch.dict( user.__salt__, dunder_salt From 6f5b4e4e5454b6f71267b6aea0d5e7bc4b7286d8 Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Mon, 26 Jun 2023 16:26:08 +0000 Subject: [PATCH 30/53] fix 3006.x nightly, this test should have already been testing for exitcode 1 not 0. as the is_leader throws an exception internal to it. --- tests/pytests/integration/reactor/test_reactor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pytests/integration/reactor/test_reactor.py b/tests/pytests/integration/reactor/test_reactor.py index 875d71623e3..0c73d282a71 100644 --- a/tests/pytests/integration/reactor/test_reactor.py +++ b/tests/pytests/integration/reactor/test_reactor.py @@ -119,21 +119,21 @@ def test_reactor_is_leader( When leader is set to false reactor should timeout/not do anything. """ ret = salt_run_cli.run("reactor.is_leader") - assert ret.returncode == 0 + assert ret.returncode == 1 assert ( "salt.exceptions.CommandExecutionError: Reactor system is not running." in ret.stdout ) ret = salt_run_cli.run("reactor.set_leader", value=True) - assert ret.returncode == 0 + assert ret.returncode == 1 assert ( "salt.exceptions.CommandExecutionError: Reactor system is not running." in ret.stdout ) ret = salt_run_cli.run("reactor.is_leader") - assert ret.returncode == 0 + assert ret.returncode == 1 assert ( "salt.exceptions.CommandExecutionError: Reactor system is not running." in ret.stdout @@ -220,7 +220,7 @@ def test_reactor_is_leader( # Let's just confirm the engine is not running once again(because the config file is deleted by now) ret = salt_run_cli.run("reactor.is_leader") - assert ret.returncode == 0 + assert ret.returncode == 1 assert ( "salt.exceptions.CommandExecutionError: Reactor system is not running." in ret.stdout From 1d852298866c7c701112e307bbc38d2ee3f58454 Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Tue, 20 Jun 2023 18:25:50 +0000 Subject: [PATCH 31/53] fix #58667 by checking for blank comps in both source and request --- changelog/58667.fixed.md | 1 + salt/modules/aptpkg.py | 4 +- .../pytests/functional/modules/test_aptpkg.py | 45 ++++++++++++++++--- 3 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 changelog/58667.fixed.md diff --git a/changelog/58667.fixed.md b/changelog/58667.fixed.md new file mode 100644 index 00000000000..c664e34e16a --- /dev/null +++ b/changelog/58667.fixed.md @@ -0,0 +1 @@ +fixes 58667 by checking for blank comps. diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index cbf9c7706c5..11e0dad6298 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -2072,7 +2072,7 @@ def del_repo(repo, **kwargs): s_comps = set(source.comps) r_comps = set(repo_comps) - if s_comps.intersection(r_comps): + if s_comps.intersection(r_comps) or (not s_comps and not r_comps): deleted_from[source.file] = 0 source.comps = list(s_comps.difference(r_comps)) if not source.comps: @@ -2093,7 +2093,7 @@ def del_repo(repo, **kwargs): s_comps = set(source.comps) r_comps = set(repo_comps) - if s_comps.intersection(r_comps): + if s_comps.intersection(r_comps) or (not s_comps and not r_comps): deleted_from[source.file] = 0 source.comps = list(s_comps.difference(r_comps)) if not source.comps: diff --git a/tests/pytests/functional/modules/test_aptpkg.py b/tests/pytests/functional/modules/test_aptpkg.py index 5bc8209f4c7..baf88a1c82f 100644 --- a/tests/pytests/functional/modules/test_aptpkg.py +++ b/tests/pytests/functional/modules/test_aptpkg.py @@ -98,6 +98,38 @@ def revert_repo_file(tmp_path): aptpkg.refresh_db() +@pytest.fixture +def build_repo_file(): + source_path = "/etc/apt/sources.list.d/source_test_list.list" + try: + test_repos = [ + "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.gpg arch=amd64] https://repo.saltproject.io/salt/py3/ubuntu/22.04/amd64/latest jammy main", + "deb http://dist.list stable/all/", + ] + with salt.utils.files.fopen(source_path, "wr+") as fp: + fp.write("\n".join(test_repos)) + yield source_path + finally: + if os.path.exists(source_path): + os.remove(source_path) + + +def get_repos_from_file(source_path): + """ + Get list of repos from repo in source_path + """ + test_repos = [] + try: + with salt.utils.files.fopen(source_path) as fp: + for line in fp: + test_repos.append(line.strip()) + except FileNotFoundError as error: + pytest.skip("Missing {}".format(error.filename)) + if not test_repos: + pytest.skip("Did not detect an APT repo") + return test_repos + + def get_current_repo(multiple_comps=False): """ Get a repo currently in sources.list @@ -195,18 +227,19 @@ def test_get_repos_doesnot_exist(): @pytest.mark.destructive_test -def test_del_repo(revert_repo_file): +def test_del_repo(build_repo_file): """ Test aptpkg.del_repo when passing repo that exists. And checking correct error is returned when it no longer exists. """ - test_repo, comps = get_current_repo() - ret = aptpkg.del_repo(repo=test_repo) - assert "Repo '{}' has been removed".format(test_repo) - with pytest.raises(salt.exceptions.CommandExecutionError) as exc: + test_repos = get_repos_from_file(build_repo_file) + for test_repo in test_repos: ret = aptpkg.del_repo(repo=test_repo) - assert "Repo {} doesn't exist".format(test_repo) in exc.value.message + assert "Repo '{}' has been removed".format(test_repo) + with pytest.raises(salt.exceptions.CommandExecutionError) as exc: + ret = aptpkg.del_repo(repo=test_repo) + assert "Repo {} doesn't exist".format(test_repo) in exc.value.message @pytest.mark.skipif( From 31428b617d93465603b217306ba3b774c24fe138 Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Tue, 20 Jun 2023 19:32:02 +0000 Subject: [PATCH 32/53] not sure why i included that r only needed w+ anyway --- tests/pytests/functional/modules/test_aptpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytests/functional/modules/test_aptpkg.py b/tests/pytests/functional/modules/test_aptpkg.py index baf88a1c82f..1d12db0a6a6 100644 --- a/tests/pytests/functional/modules/test_aptpkg.py +++ b/tests/pytests/functional/modules/test_aptpkg.py @@ -106,7 +106,7 @@ def build_repo_file(): "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.gpg arch=amd64] https://repo.saltproject.io/salt/py3/ubuntu/22.04/amd64/latest jammy main", "deb http://dist.list stable/all/", ] - with salt.utils.files.fopen(source_path, "wr+") as fp: + with salt.utils.files.fopen(source_path, "w+") as fp: fp.write("\n".join(test_repos)) yield source_path finally: From dc83ef63ba5bf5a39e12e60e703c4d3dcbc61ce5 Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Tue, 20 Jun 2023 12:38:37 -0700 Subject: [PATCH 33/53] Update changelog/58667.fixed.md Co-authored-by: Megan Wilhite --- changelog/58667.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/58667.fixed.md b/changelog/58667.fixed.md index c664e34e16a..ebc2cb617cd 100644 --- a/changelog/58667.fixed.md +++ b/changelog/58667.fixed.md @@ -1 +1 @@ -fixes 58667 by checking for blank comps. +fixes aptpkg module by checking for blank comps. From b265eaecb74ee4963a3d4ba4968a5154e6a0fa6a Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Wed, 21 Jun 2023 21:08:20 +0000 Subject: [PATCH 34/53] add skip_if_not_root to destructive tests in aptpkg as they all manipulate the filesystem. --- tests/pytests/functional/modules/test_aptpkg.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/pytests/functional/modules/test_aptpkg.py b/tests/pytests/functional/modules/test_aptpkg.py index 1d12db0a6a6..71693655c60 100644 --- a/tests/pytests/functional/modules/test_aptpkg.py +++ b/tests/pytests/functional/modules/test_aptpkg.py @@ -85,6 +85,8 @@ def configure_loader_modules(minion_opts): @pytest.fixture() +@pytest.mark.destructive_test +@pytest.mark.skip_if_not_root def revert_repo_file(tmp_path): try: repo_file = pathlib.Path("/etc") / "apt" / "sources.list" @@ -99,6 +101,8 @@ def revert_repo_file(tmp_path): @pytest.fixture +@pytest.mark.destructive_test +@pytest.mark.skip_if_not_root def build_repo_file(): source_path = "/etc/apt/sources.list.d/source_test_list.list" try: @@ -227,6 +231,7 @@ def test_get_repos_doesnot_exist(): @pytest.mark.destructive_test +@pytest.mark.skip_if_not_root def test_del_repo(build_repo_file): """ Test aptpkg.del_repo when passing repo @@ -275,6 +280,7 @@ def test__expand_repo_def(grains): @pytest.mark.destructive_test +@pytest.mark.skip_if_not_root def test_mod_repo(revert_repo_file): """ Test aptpkg.mod_repo when the repo exists. @@ -289,6 +295,7 @@ def test_mod_repo(revert_repo_file): @pytest.mark.destructive_test +@pytest.mark.skip_if_not_root def test_mod_repo_no_file(tmp_path, revert_repo_file): """ Test aptpkg.mod_repo when the file does not exist. @@ -317,6 +324,7 @@ def add_key(request, get_key_file): @pytest.mark.parametrize("get_key_file", KEY_FILES, indirect=True) @pytest.mark.parametrize("add_key", [False, True], indirect=True) @pytest.mark.destructive_test +@pytest.mark.skip_if_not_root def test_get_repo_keys(get_key_file, add_key): """ Test aptpkg.get_repo_keys when aptkey is False and True @@ -330,6 +338,7 @@ def test_get_repo_keys(get_key_file, add_key): @pytest.mark.parametrize("key", [False, True]) @pytest.mark.destructive_test +@pytest.mark.skip_if_not_root def test_get_repo_keys_keydir_not_exist(key): """ Test aptpkg.get_repo_keys when aptkey is False and True From 52f965eb42f75815599e768f61667aaacc371c5c Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Thu, 22 Jun 2023 00:17:36 +0000 Subject: [PATCH 35/53] remove destructive and skip_if_not_root from pytest.fixture --- tests/pytests/functional/modules/test_aptpkg.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/pytests/functional/modules/test_aptpkg.py b/tests/pytests/functional/modules/test_aptpkg.py index 71693655c60..3e89ed700b1 100644 --- a/tests/pytests/functional/modules/test_aptpkg.py +++ b/tests/pytests/functional/modules/test_aptpkg.py @@ -85,8 +85,6 @@ def configure_loader_modules(minion_opts): @pytest.fixture() -@pytest.mark.destructive_test -@pytest.mark.skip_if_not_root def revert_repo_file(tmp_path): try: repo_file = pathlib.Path("/etc") / "apt" / "sources.list" From 90b3c25364f9388b3e967b03bfdbb4c33e4dbd4e Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Thu, 22 Jun 2023 22:58:24 +0000 Subject: [PATCH 36/53] remove destructive and skip_if_not_root from fixture --- tests/pytests/functional/modules/test_aptpkg.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/pytests/functional/modules/test_aptpkg.py b/tests/pytests/functional/modules/test_aptpkg.py index 3e89ed700b1..7e086e498e2 100644 --- a/tests/pytests/functional/modules/test_aptpkg.py +++ b/tests/pytests/functional/modules/test_aptpkg.py @@ -99,8 +99,6 @@ def revert_repo_file(tmp_path): @pytest.fixture -@pytest.mark.destructive_test -@pytest.mark.skip_if_not_root def build_repo_file(): source_path = "/etc/apt/sources.list.d/source_test_list.list" try: From 021c6792bfe655071e80864acad21e27f762cc10 Mon Sep 17 00:00:00 2001 From: Thomas Phipps Date: Mon, 26 Jun 2023 19:11:54 +0000 Subject: [PATCH 37/53] fix the runner.run to have its own full_return that defautls to False so anything currently using runner.run will work the same as before. this is what i should have done from the start. --- salt/cli/run.py | 2 +- salt/runner.py | 4 ++-- tests/support/case.py | 6 ------ 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/salt/cli/run.py b/salt/cli/run.py index c6f6f42c14e..fba70fffbec 100644 --- a/salt/cli/run.py +++ b/salt/cli/run.py @@ -31,7 +31,7 @@ class SaltRun(salt.utils.parsers.SaltRunOptionParser): if check_user(self.config["user"]): pr = salt.utils.profile.activate_profile(profiling_enabled) try: - ret = runner.run() + ret = runner.run(full_return=True) # In older versions ret['data']['retcode'] was used # for signaling the return code. This has been # changed for the orchestrate runner, but external diff --git a/salt/runner.py b/salt/runner.py index d390af93cde..2a19636b8ed 100644 --- a/salt/runner.py +++ b/salt/runner.py @@ -207,7 +207,7 @@ class Runner(RunnerClient): print(docs[fun]) # TODO: move to mixin whenever we want a salt-wheel cli - def run(self): + def run(self, full_return=False): """ Execute the runner sequence """ @@ -306,7 +306,7 @@ class Runner(RunnerClient): tag=async_pub["tag"], jid=async_pub["jid"], daemonize=False, - full_return=True, + full_return=full_return, ) except salt.exceptions.SaltException as exc: with salt.utils.event.get_event("master", opts=self.opts) as evt: diff --git a/tests/support/case.py b/tests/support/case.py index 4b3222877b1..55b537f6ffd 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -191,12 +191,6 @@ class ShellCase(TestCase, AdaptedConfigurationTestCaseMixin, ScriptPathMixin): with RedirectStdStreams(): runner = salt.runner.Runner(opts) ret["return"] = runner.run() - if ( - isinstance(ret["return"], dict) - and "return" in ret["return"] - and "retcode" not in ret["return"] - ): - ret["return"] = ret["return"]["return"] try: ret["jid"] = runner.jid except AttributeError: From 5bad773949ae4fc4c90f926e6d3f70c9e8dec29b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 27 Jun 2023 09:38:37 +0100 Subject: [PATCH 38/53] Don't override the `on` jinja block on the CI workflow template Signed-off-by: Pedro Algarvio --- .github/workflows/ci.yml | 9 ++++++++- .github/workflows/templates/ci.yml.jinja | 6 ------ changelog/64547.added.md | 4 ++++ 3 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 changelog/64547.added.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11db8968e25..0763c5176ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,9 +3,16 @@ --- name: CI run-name: "CI (${{ github.event_name == 'pull_request' && format('pr: #{0}', github.event.number) || format('{0}: {1}', startsWith(github.event.ref, 'refs/tags') && 'tag' || 'branch', github.ref_name) }})" + on: push: {} - pull_request: {} + pull_request: + types: + - labeled + - unlabeled + - opened + - reopened + - synchronize env: COLUMNS: 190 diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 809b3c05412..5faeabf91f4 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -1,12 +1,6 @@ <%- extends 'layout.yml.jinja' %> <%- set pre_commit_version = "3.0.4" %> -<%- block on %> -on: - push: {} - pull_request: {} -<%- endblock on %> - <%- block jobs %> <{- super() }> diff --git a/changelog/64547.added.md b/changelog/64547.added.md new file mode 100644 index 00000000000..a28c6314215 --- /dev/null +++ b/changelog/64547.added.md @@ -0,0 +1,4 @@ +Several fixes to the CI workflow: + +* Don't override the `on` Jinja block on the `ci.yaml` template. This enables reacting to labels getting added/removed + to/from pull requests. From 2d513978a5d4b1a578fe89d6622c2f79e1292220 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 26 Jun 2023 18:21:35 +0100 Subject: [PATCH 39/53] Switch to using `tools` and re-use the event payload available instead of querying again Signed-off-by: Pedro Algarvio --- .github/actions/get-pull-labels/action.yml | 23 ---- .github/actions/get-pull-number/action.yml | 46 -------- .github/actions/get-pull-request/action.yml | 30 ----- .github/workflows/ci.yml | 33 +----- .github/workflows/nightly.yml | 33 +----- .github/workflows/scheduled.yml | 33 +----- .github/workflows/staging.yml | 33 +----- .github/workflows/templates/layout.yml.jinja | 37 ++----- changelog/64547.added.md | 2 + tools/ci.py | 109 ++++++++++++++++++- 10 files changed, 138 insertions(+), 241 deletions(-) delete mode 100644 .github/actions/get-pull-labels/action.yml delete mode 100644 .github/actions/get-pull-number/action.yml delete mode 100644 .github/actions/get-pull-request/action.yml diff --git a/.github/actions/get-pull-labels/action.yml b/.github/actions/get-pull-labels/action.yml deleted file mode 100644 index 2da0a2c9dae..00000000000 --- a/.github/actions/get-pull-labels/action.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -name: get-pull-labels -description: Get Pull Labels -inputs: - pull-request: - type: string - -outputs: - labels: - value: ${{ steps.get-pull-labels.outputs.labels }} - -runs: - using: composite - steps: - - name: Get Pull Labels - id: get-pull-labels - shell: bash - env: - PULL_REQUEST: ${{ inputs.pull-request }} - run: | - labels=$(jq -c '[.labels[].name]' <<< $PULL_REQUEST) - echo $labels - echo "labels=$labels" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/get-pull-number/action.yml b/.github/actions/get-pull-number/action.yml deleted file mode 100644 index 00fd0425aff..00000000000 --- a/.github/actions/get-pull-number/action.yml +++ /dev/null @@ -1,46 +0,0 @@ - ---- -name: get-pull-number -description: Get Pull Number -inputs: - owner: - type: string - repo: - type: string - sha: - type: string - pull-number: - default: null - -outputs: - number: - value: ${{ steps.get-pull-number.outputs.number }} - -runs: - using: composite - steps: - - name: Get Pull Number - id: get-pull-number - shell: bash - env: - GITHUB_OWNER: ${{ inputs.owner }} - GITHUB_REPO: ${{ inputs.repo }} - GITHUB_SHA: ${{ inputs.sha }} - GITHUB_PULL_NUMBER: ${{ inputs.pull-number }} - run: | - if [ -z "$GITHUB_PULL_NUMBER" ] - then - echo "Searching For Pull Number" - echo $GITHUB_OWNER - echo $GITHUB_REPO - echo $GITHUB_SHA - pulls=$(gh api repos/$GITHUB_OWNER/$GITHUB_REPO/commits/$GITHUB_SHA/pulls) - echo $pulls - full_name=$GITHUB_OWNER/$GITHUB_REPO - number=$(jq -c --arg r "$full_name" '[.[] | select(.url | contains($r))][0].number' <<< $pulls ) - else - echo "Given Pull Number" - number=$GITHUB_PULL_NUMBER - fi - echo $number - echo "number=$number" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/get-pull-request/action.yml b/.github/actions/get-pull-request/action.yml deleted file mode 100644 index 781aa24fe98..00000000000 --- a/.github/actions/get-pull-request/action.yml +++ /dev/null @@ -1,30 +0,0 @@ - ---- -name: get-pull-request -description: Get Pull Request -inputs: - owner: - type: string - repo: - type: string - pull-number: - type: number - -outputs: - pull-request: - value: ${{ steps.get-pull-request.outputs.request }} - -runs: - using: composite - steps: - - name: Get Pull Request - id: get-pull-request - shell: bash - env: - GITHUB_OWNER: ${{ inputs.owner }} - GITHUB_REPO: ${{ inputs.repo }} - GITHUB_PULL_NUMBER: ${{ inputs.pull-number }} - run: | - pull=$(gh api repos/$GITHUB_OWNER/$GITHUB_REPO/pulls/$GITHUB_PULL_NUMBER) - echo $pull - echo "request=$pull" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0763c5176ee..51970256ae0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,35 +146,12 @@ jobs: salt-version: "" validate-version: true - - name: Get Pull Number - if: ${{ github.event_name == 'pull_request' }} - id: get-pull-number - uses: ./.github/actions/get-pull-number - with: - owner: ${{ github.repository_owner }} - repo: ${{ github.event.repository.name }} - sha: ${{ github.sha }} - pull-number: ${{ github.event.pull_request.number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Pull Request - if: ${{ github.event_name == 'pull_request' }} - id: get-pull-request - uses: ./.github/actions/get-pull-request - with: - owner: ${{ github.repository_owner }} - repo: ${{ github.event.repository.name }} - pull-number: ${{ steps.get-pull-number.outputs.number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Pull Labels - if: ${{ github.event_name == 'pull_request' }} + - name: Get Pull Request Test Labels id: get-pull-labels - uses: ./.github/actions/get-pull-labels - with: - pull-request: ${{ steps.get-pull-request.outputs.pull-request }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + tools ci get-pr-test-labels --repository ${{ github.repository }} - name: Write Changed Files To A Local File run: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a5371f7878b..6dd4f84bfd8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -192,35 +192,12 @@ jobs: salt-version: "" validate-version: true - - name: Get Pull Number - if: ${{ github.event_name == 'pull_request' }} - id: get-pull-number - uses: ./.github/actions/get-pull-number - with: - owner: ${{ github.repository_owner }} - repo: ${{ github.event.repository.name }} - sha: ${{ github.sha }} - pull-number: ${{ github.event.pull_request.number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Pull Request - if: ${{ github.event_name == 'pull_request' }} - id: get-pull-request - uses: ./.github/actions/get-pull-request - with: - owner: ${{ github.repository_owner }} - repo: ${{ github.event.repository.name }} - pull-number: ${{ steps.get-pull-number.outputs.number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Pull Labels - if: ${{ github.event_name == 'pull_request' }} + - name: Get Pull Request Test Labels id: get-pull-labels - uses: ./.github/actions/get-pull-labels - with: - pull-request: ${{ steps.get-pull-request.outputs.pull-request }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + tools ci get-pr-test-labels --repository ${{ github.repository }} - name: Write Changed Files To A Local File run: diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 2597688b0c3..242560a992c 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -182,35 +182,12 @@ jobs: salt-version: "" validate-version: true - - name: Get Pull Number - if: ${{ github.event_name == 'pull_request' }} - id: get-pull-number - uses: ./.github/actions/get-pull-number - with: - owner: ${{ github.repository_owner }} - repo: ${{ github.event.repository.name }} - sha: ${{ github.sha }} - pull-number: ${{ github.event.pull_request.number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Pull Request - if: ${{ github.event_name == 'pull_request' }} - id: get-pull-request - uses: ./.github/actions/get-pull-request - with: - owner: ${{ github.repository_owner }} - repo: ${{ github.event.repository.name }} - pull-number: ${{ steps.get-pull-number.outputs.number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Pull Labels - if: ${{ github.event_name == 'pull_request' }} + - name: Get Pull Request Test Labels id: get-pull-labels - uses: ./.github/actions/get-pull-labels - with: - pull-request: ${{ steps.get-pull-request.outputs.pull-request }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + tools ci get-pr-test-labels --repository ${{ github.repository }} - name: Write Changed Files To A Local File run: diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index ff5a727408e..f883cba1e43 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -172,35 +172,12 @@ jobs: salt-version: "${{ inputs.salt-version }}" validate-version: true - - name: Get Pull Number - if: ${{ github.event_name == 'pull_request' }} - id: get-pull-number - uses: ./.github/actions/get-pull-number - with: - owner: ${{ github.repository_owner }} - repo: ${{ github.event.repository.name }} - sha: ${{ github.sha }} - pull-number: ${{ github.event.pull_request.number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Pull Request - if: ${{ github.event_name == 'pull_request' }} - id: get-pull-request - uses: ./.github/actions/get-pull-request - with: - owner: ${{ github.repository_owner }} - repo: ${{ github.event.repository.name }} - pull-number: ${{ steps.get-pull-number.outputs.number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Pull Labels - if: ${{ github.event_name == 'pull_request' }} + - name: Get Pull Request Test Labels id: get-pull-labels - uses: ./.github/actions/get-pull-labels - with: - pull-request: ${{ steps.get-pull-request.outputs.pull-request }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + tools ci get-pr-test-labels --repository ${{ github.repository }} - name: Check Existing Releases env: diff --git a/.github/workflows/templates/layout.yml.jinja b/.github/workflows/templates/layout.yml.jinja index 855c8a78654..9ca3bf76fa8 100644 --- a/.github/workflows/templates/layout.yml.jinja +++ b/.github/workflows/templates/layout.yml.jinja @@ -191,35 +191,12 @@ jobs: salt-version: "<{ prepare_workflow_salt_version_input }>" validate-version: true - - name: Get Pull Number - if: ${{ github.event_name == 'pull_request' }} - id: get-pull-number - uses: ./.github/actions/get-pull-number - with: - owner: ${{ github.repository_owner }} - repo: ${{ github.event.repository.name }} - sha: ${{ github.sha }} - pull-number: ${{ github.event.pull_request.number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Pull Request - if: ${{ github.event_name == 'pull_request' }} - id: get-pull-request - uses: ./.github/actions/get-pull-request - with: - owner: ${{ github.repository_owner }} - repo: ${{ github.event.repository.name }} - pull-number: ${{ steps.get-pull-number.outputs.number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Pull Labels - if: ${{ github.event_name == 'pull_request' }} + - name: Get Pull Request Test Labels id: get-pull-labels - uses: ./.github/actions/get-pull-labels - with: - pull-request: ${{ steps.get-pull-request.outputs.pull-request }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + tools ci get-pr-test-labels --repository ${{ github.repository }} <%- if prepare_actual_release %> @@ -262,7 +239,9 @@ jobs: - name: Define Jobs id: define-jobs run: | - tools ci define-jobs<{ prepare_workflow_skip_test_suite }><{ prepare_workflow_skip_pkg_test_suite }><{ prepare_workflow_skip_pkg_download_test_suite }> ${{ github.event_name }} changed-files.json + tools ci define-jobs<{ prepare_workflow_skip_test_suite }><{ + prepare_workflow_skip_pkg_test_suite }><{ prepare_workflow_skip_pkg_download_test_suite + }> ${{ github.event_name }} changed-files.json - name: Check Defined Jobs run: | diff --git a/changelog/64547.added.md b/changelog/64547.added.md index a28c6314215..daa1edaa0d6 100644 --- a/changelog/64547.added.md +++ b/changelog/64547.added.md @@ -2,3 +2,5 @@ Several fixes to the CI workflow: * Don't override the `on` Jinja block on the `ci.yaml` template. This enables reacting to labels getting added/removed to/from pull requests. +* Switch to using `tools` and re-use the event payload available instead of querying the GH API again to get the pull + request labels diff --git a/tools/ci.py b/tools/ci.py index ba7a7c2f849..c75310dd09c 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -9,7 +9,7 @@ import logging import os import pathlib import time -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from ptscripts import Context, command_group @@ -672,3 +672,110 @@ def get_releases(ctx: Context, repository: str = "saltstack/salt"): wfh.write(f"latest-release={latest}\n") wfh.write(f"releases={json.dumps(str_releases)}\n") ctx.exit(0) + + +@ci.command( + name="get-pr-test-labels", + arguments={ + "pr": { + "help": "Pull request number", + }, + "repository": { + "help": "Github repository.", + }, + }, +) +def get_pr_test_labels( + ctx: Context, repository: str = "saltstack/salt", pr: int = None +): + """ + Set the pull-request labels. + """ + gh_event_path = os.environ.get("GITHUB_EVENT_PATH") or None + if gh_event_path is None: + labels = _get_pr_test_labels_from_api(ctx, repository, pr=pr) + else: + if TYPE_CHECKING: + assert gh_event_path is not None + + try: + gh_event = json.loads(open(gh_event_path).read()) + except Exception as exc: + ctx.error( + f"Could not load the GH Event payload from {gh_event_path!r}:\n", exc + ) + ctx.exit(1) + + if "pull_request" not in gh_event: + ctx.warning("The 'pull_request' key was not found on the event payload.") + ctx.exit(1) + + pr = gh_event["pull_request"]["number"] + labels = _get_pr_test_labels_from_event_payload(gh_event) + + if labels: + ctx.info(f"Test labels for pull-request #{pr} on {repository}:") + for name, description in labels: + ctx.info(f" * [yellow]{name}[/yellow]: {description}") + else: + ctx.info(f"No test labels for pull-request #{pr} on {repository}") + + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output is None: + ctx.exit(0) + + if TYPE_CHECKING: + assert github_output is not None + + ctx.info("Writing 'labels' to the github outputs file") + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"labels={json.dumps([label[0] for label in labels])}\n") + ctx.exit(0) + + +def _get_pr_test_labels_from_api( + ctx: Context, repository: str = "saltstack/salt", pr: int = None +) -> list[tuple[str, str]]: + """ + Set the pull-request labels. + """ + if pr is None: + ctx.error( + "Could not find the 'GITHUB_EVENT_PATH' variable and the " + "--pr flag was not passed. Unable to detect pull-request number." + ) + ctx.exit(1) + with ctx.web as web: + headers = { + "Accept": "application/vnd.github+json", + } + if "GITHUB_TOKEN" in os.environ: + headers["Authorization"] = f"Bearer {os.environ['GITHUB_TOKEN']}" + web.headers.update(headers) + ret = web.get(f"https://api.github.com/repos/{repository}/pulls/{pr}") + if ret.status_code != 200: + ctx.error( + f"Failed to get the #{pr} pull-request details on repository {repository!r}: {ret.reason}" + ) + ctx.exit(1) + pr_details = ret.json() + return _filter_test_labels(pr_details["labels"]) + + +def _get_pr_test_labels_from_event_payload( + gh_event: dict[str, Any] +) -> list[tuple[str, str]]: + """ + Get the pull-request test labels. + """ + if "pull_request" not in gh_event: + return [] + return _filter_test_labels(gh_event["pull_request"]["labels"]) + + +def _filter_test_labels(labels: list[dict[str, Any]]) -> list[tuple[str, str]]: + return [ + (label["name"], label["description"]) + for label in labels + if label["name"].startswith("test:") + ] From 3fe2f449031e8924881b349127e34a32fb43d50d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 27 Jun 2023 10:26:08 +0100 Subject: [PATCH 40/53] Choose a full test run when a PR has the `test:full` label Fixes #64539 Signed-off-by: Pedro Algarvio --- changelog/64539.fixed.md | 1 + tools/ci.py | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 changelog/64539.fixed.md diff --git a/changelog/64539.fixed.md b/changelog/64539.fixed.md new file mode 100644 index 00000000000..38c3af47edf --- /dev/null +++ b/changelog/64539.fixed.md @@ -0,0 +1 @@ +Added a `test:full` label in the salt repository, which, when selected, will force a full test run. diff --git a/tools/ci.py b/tools/ci.py index c75310dd09c..c49b3f8e02f 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -439,8 +439,23 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): ctx.error(f"Could not load the changed files from '{changed_files}': {exc}") ctx.exit(1) + labels: list[str] = [] + gh_event_path = os.environ.get("GITHUB_EVENT_PATH") or None + if gh_event_path is not None: + try: + gh_event = json.loads(open(gh_event_path).read()) + except Exception as exc: + ctx.error( + f"Could not load the GH Event payload from {gh_event_path!r}:\n", exc + ) + ctx.exit(1) + + labels.extend( + label[0] for label in _get_pr_test_labels_from_event_payload(gh_event) + ) + # So, it's a pull request... - # Based on which files changed, or other things like PR comments we can + # Based on which files changed, or other things like PR labels we can # decide what to run, or even if the full test run should be running on the # pull request, etc... changed_pkg_requirements_files = json.loads( @@ -471,6 +486,10 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): wfh.write(f"{path}\n") wfh.write("\n\n") testrun = {"type": "full"} + elif "test:full" in labels: + with open(github_step_summary, "a", encoding="utf-8") as wfh: + wfh.write("Full test run chosen because the label `test:full` is set.\n") + testrun = {"type": "full"} else: testrun_changed_files_path = tools.utils.REPO_ROOT / "testrun-changed-files.txt" testrun = { From 4b85fc6e17224f8ea231620d3e53a6a5e2d3321d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 27 Jun 2023 12:35:02 +0100 Subject: [PATCH 41/53] Concentrate test selection to a single place Signed-off-by: Pedro Algarvio --- .github/workflows/ci.yml | 23 --------- .github/workflows/nightly.yml | 23 --------- .github/workflows/scheduled.yml | 23 --------- .github/workflows/staging.yml | 23 --------- .../workflows/templates/test-salt.yml.jinja | 3 -- .github/workflows/test-action-macos.yml | 46 +++-------------- .github/workflows/test-action.yml | 46 +++-------------- changelog/64547.added.md | 1 + tools/ci.py | 50 ++++++++++++++++--- 9 files changed, 57 insertions(+), 181 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51970256ae0..baf729efce6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -854,7 +854,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -873,7 +872,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -892,7 +890,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -911,7 +908,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -930,7 +926,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -949,7 +944,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -968,7 +962,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -987,7 +980,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1006,7 +998,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1025,7 +1016,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1044,7 +1034,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1063,7 +1052,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1082,7 +1070,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1101,7 +1088,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1120,7 +1106,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1139,7 +1124,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1158,7 +1142,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1177,7 +1160,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1196,7 +1178,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1215,7 +1196,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1234,7 +1214,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1253,7 +1232,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} @@ -1272,7 +1250,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: ${{ github.event_name == 'pull_request' }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6dd4f84bfd8..0bde1d78e3c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -911,7 +911,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -930,7 +929,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -949,7 +947,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -968,7 +965,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -987,7 +983,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1006,7 +1001,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1025,7 +1019,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1044,7 +1037,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1063,7 +1055,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1082,7 +1073,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1101,7 +1091,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1120,7 +1109,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1139,7 +1127,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1158,7 +1145,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1177,7 +1163,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1196,7 +1181,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1215,7 +1199,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1234,7 +1217,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1253,7 +1235,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1272,7 +1253,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1291,7 +1271,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1310,7 +1289,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1329,7 +1307,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 242560a992c..603f2d4c579 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -890,7 +890,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -909,7 +908,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -928,7 +926,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -947,7 +944,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -966,7 +962,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -985,7 +980,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1004,7 +998,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1023,7 +1016,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1042,7 +1034,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1061,7 +1052,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1080,7 +1070,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1099,7 +1088,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1118,7 +1106,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1137,7 +1124,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1156,7 +1142,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1175,7 +1160,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1194,7 +1178,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1213,7 +1196,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1232,7 +1214,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1251,7 +1232,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1270,7 +1250,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1289,7 +1268,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false @@ -1308,7 +1286,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: false skip-junit-reports: false diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index f883cba1e43..7eab1044545 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -896,7 +896,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -915,7 +914,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -934,7 +932,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -953,7 +950,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -972,7 +968,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -991,7 +986,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1010,7 +1004,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1029,7 +1022,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1048,7 +1040,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1067,7 +1058,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1086,7 +1076,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1105,7 +1094,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1124,7 +1112,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1143,7 +1130,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1162,7 +1148,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1181,7 +1166,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1200,7 +1184,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1219,7 +1202,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1238,7 +1220,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1257,7 +1238,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1276,7 +1256,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1295,7 +1274,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true @@ -1314,7 +1292,6 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: true skip-junit-reports: true diff --git a/.github/workflows/templates/test-salt.yml.jinja b/.github/workflows/templates/test-salt.yml.jinja index 0d6d3557dcf..d54ab8181ee 100644 --- a/.github/workflows/templates/test-salt.yml.jinja +++ b/.github/workflows/templates/test-salt.yml.jinja @@ -19,7 +19,6 @@ testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }> - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: <{ skip_test_coverage_check }> skip-junit-reports: <{ skip_junit_reports_check }> @@ -43,7 +42,6 @@ testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }> - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: <{ skip_test_coverage_check }> skip-junit-reports: <{ skip_junit_reports_check }> @@ -85,7 +83,6 @@ testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }> - pull-labels: ${{ needs.prepare-workflow.outputs.pull-labels }} skip-code-coverage: <{ skip_test_coverage_check }> skip-junit-reports: <{ skip_junit_reports_check }> diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index b8088c64522..3d1edefdf96 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -42,11 +42,6 @@ on: type: string description: The onedir package name to use default: salt - pull-labels: - required: false - type: string - description: List of all the pull labels - default: '["test:slow", "test:core"]' skip-code-coverage: required: false type: boolean @@ -269,36 +264,9 @@ jobs: run: | sudo -E nox -e ${{ env.NOX_SESSION }} -- ${{ matrix.tests-chunk }} -- -k "mac or darwin" - - name: Get Test Flags - id: get-test-flags - shell: bash - env: - PULL_LABELS: ${{ inputs.pull-labels }} - run: | - echo "$PULL_LABELS" - # shellcheck disable=SC2086 - no_fast_tests="$(jq -c '. | any(index("test:no-fast"))' <<< $PULL_LABELS)" - # shellcheck disable=SC2086 - slow_tests="$(jq -c '. | any(index("test:slow"))' <<< $PULL_LABELS)" - # shellcheck disable=SC2086 - core_tests="$(jq -c '. | any(index("test:core"))' <<< $PULL_LABELS)" - # shellcheck disable=SC2086 - flaky_jail_tests="$(jq -c '. | any(index("test:flaky-jail"))' <<< $PULL_LABELS)" - echo "$no_fast_tests" - echo "$slow_tests" - echo "$core_tests" - echo "$flaky_jail_tests" - # shellcheck disable=SC2086 - { - echo "no_fast_tests=$no_fast_tests"; - echo "slow_tests=$slow_tests"; - echo "core_tests=$core_tests"; - echo "flaky_jail_tests=$flaky_jail_tests"; - } >> "$GITHUB_OUTPUT" - - name: Run Fast/Changed Tests id: run-fast-changed-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.no_fast_tests == 'true' }} + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] == false }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -315,7 +283,7 @@ jobs: - name: Run Slow/Changed Tests id: run-slow-changed-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.slow_tests == 'false' }} + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] == false }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -332,7 +300,7 @@ jobs: - name: Run Core/Changed Tests id: run-core-changed-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.core_tests == 'false' }} + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] == false }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -349,7 +317,7 @@ jobs: - name: Run Fast Tests id: run-fast-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.no_fast_tests == 'false' }} + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -365,7 +333,7 @@ jobs: - name: Run Slow Tests id: run-slow-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.slow_tests == 'true' }} + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -381,7 +349,7 @@ jobs: - name: Run Core Tests id: run-core-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.core_tests == 'true' }} + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -397,7 +365,7 @@ jobs: - name: Run Flaky Tests id: run-flaky-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.flaky_jail_tests == 'true' }} + if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 0982e7446eb..49d75dcd86b 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -37,11 +37,6 @@ on: type: string description: The onedir package name to use default: salt - pull-labels: - required: false - type: string - description: List of all the pull labels - default: '["test:slow", "test:core"]' skip-code-coverage: required: false type: boolean @@ -303,36 +298,9 @@ jobs: --nox-session=${{ env.NOX_SESSION }} ${{ inputs.distro-slug }} \ ${{ matrix.tests-chunk }} - - name: Get Test Flags - id: get-test-flags - shell: bash - env: - PULL_LABELS: ${{ inputs.pull-labels }} - run: | - echo "$PULL_LABELS" - # shellcheck disable=SC2086 - no_fast_tests="$(jq -c '. | any(index("test:no-fast"))' <<< $PULL_LABELS)" - # shellcheck disable=SC2086 - slow_tests="$(jq -c '. | any(index("test:slow"))' <<< $PULL_LABELS)" - # shellcheck disable=SC2086 - core_tests="$(jq -c '. | any(index("test:core"))' <<< $PULL_LABELS)" - # shellcheck disable=SC2086 - flaky_jail_tests="$(jq -c '. | any(index("test:flaky-jail"))' <<< $PULL_LABELS)" - echo "$no_fast_tests" - echo "$slow_tests" - echo "$core_tests" - echo "$flaky_jail_tests" - # shellcheck disable=SC2086 - { - echo "no_fast_tests=$no_fast_tests"; - echo "slow_tests=$slow_tests"; - echo "core_tests=$core_tests"; - echo "flaky_jail_tests=$flaky_jail_tests"; - } >> "$GITHUB_OUTPUT" - - name: Run Fast/Changed Tests id: run-fast-changed-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.no_fast_tests == 'true' }} + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] == false }} run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ --nox-session=${{ env.NOX_SESSION }} --rerun-failures ${{ inputs.distro-slug }} \ @@ -341,7 +309,7 @@ jobs: - name: Run Slow/Changed Tests id: run-slow-changed-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.slow_tests == 'false' }} + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] == false }} run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ --nox-session=${{ env.NOX_SESSION }} --rerun-failures ${{ inputs.distro-slug }} \ @@ -350,7 +318,7 @@ jobs: - name: Run Core/Changed Tests id: run-core-changed-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.core_tests == 'false' }} + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] == false }} run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ --nox-session=${{ env.NOX_SESSION }} --rerun-failures ${{ inputs.distro-slug }} \ @@ -359,7 +327,7 @@ jobs: - name: Run Fast Tests id: run-fast-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.no_fast_tests == 'false' }} + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }} run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ --nox-session=${{ env.NOX_SESSION }} --rerun-failures ${{ (inputs.skip-code-coverage && matrix.tests-chunk != 'unit') && '--skip-code-coverage' || '' }} \ @@ -367,7 +335,7 @@ jobs: - name: Run Slow Tests id: run-slow-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.slow_tests == 'true' }} + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }} run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ --nox-session=${{ env.NOX_SESSION }} --rerun-failures ${{ inputs.distro-slug }} \ @@ -375,7 +343,7 @@ jobs: - name: Run Core Tests id: run-core-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.core_tests == 'true' }} + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }} run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ --nox-session=${{ env.NOX_SESSION }} --rerun-failures ${{ inputs.distro-slug }} \ @@ -383,7 +351,7 @@ jobs: - name: Run Flaky Tests id: run-flaky-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && steps.get-test-flags.outputs.flaky_jail_tests == 'true' }} + if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }} run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ --nox-session=${{ env.NOX_SESSION }} --rerun-failures ${{ inputs.distro-slug }} \ diff --git a/changelog/64547.added.md b/changelog/64547.added.md index daa1edaa0d6..f1bc6e6a131 100644 --- a/changelog/64547.added.md +++ b/changelog/64547.added.md @@ -4,3 +4,4 @@ Several fixes to the CI workflow: to/from pull requests. * Switch to using `tools` and re-use the event payload available instead of querying the GH API again to get the pull request labels +* Concentrate test selection by labels to a single place diff --git a/tools/ci.py b/tools/ci.py index c49b3f8e02f..c3f79aab355 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -8,6 +8,7 @@ import json import logging import os import pathlib +import sys import time from typing import TYPE_CHECKING, Any @@ -15,6 +16,11 @@ from ptscripts import Context, command_group import tools.utils +if sys.version_info < (3, 11): + from typing_extensions import NotRequired, TypedDict +else: + from typing import NotRequired, TypedDict # pylint: disable=no-name-in-module + log = logging.getLogger(__name__) # Define the command group @@ -381,6 +387,12 @@ def define_jobs( wfh.write(f"jobs={json.dumps(jobs)}\n") +class TestRun(TypedDict): + type: str + from_filenames: NotRequired[str] + selected_tests: NotRequired[dict[str, bool]] + + @ci.command( name="define-testrun", arguments={ @@ -418,7 +430,7 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): if event_name != "pull_request": # In this case, a full test run is in order ctx.info("Writing 'testrun' to the github outputs file") - testrun = {"type": "full"} + testrun = TestRun(type="full") with open(github_output, "a", encoding="utf-8") as wfh: wfh.write(f"testrun={json.dumps(testrun)}\n") @@ -470,7 +482,7 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): "Full test run chosen because there was a change made " "to `cicd/golden-images.json`.\n" ) - testrun = {"type": "full"} + testrun = TestRun(type="full") elif changed_pkg_requirements_files or changed_test_requirements_files: with open(github_step_summary, "a", encoding="utf-8") as wfh: wfh.write( @@ -485,19 +497,19 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): ): wfh.write(f"{path}\n") wfh.write("\n\n") - testrun = {"type": "full"} + testrun = TestRun(type="full") elif "test:full" in labels: with open(github_step_summary, "a", encoding="utf-8") as wfh: wfh.write("Full test run chosen because the label `test:full` is set.\n") - testrun = {"type": "full"} + testrun = TestRun(type="full") else: testrun_changed_files_path = tools.utils.REPO_ROOT / "testrun-changed-files.txt" - testrun = { - "type": "changed", - "from-filenames": str( + testrun = TestRun( + type="changed", + from_filenames=str( testrun_changed_files_path.relative_to(tools.utils.REPO_ROOT) ), - } + ) ctx.info(f"Writing {testrun_changed_files_path.name} ...") selected_changed_files = [] for fpath in json.loads(changed_files_contents["testrun_files"]): @@ -517,6 +529,28 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): if testrun["type"] == "changed": with open(github_step_summary, "a", encoding="utf-8") as wfh: wfh.write("Partial test run chosen.\n") + testrun["selected_tests"] = { + "core": False, + "slow": False, + "fast": True, + "flaky": False, + } + if "test:slow" in labels: + with open(github_step_summary, "a", encoding="utf-8") as wfh: + wfh.write("Slow tests chosen by `test:slow` label.\n") + testrun["selected_tests"]["slow"] = True + if "test:core" in labels: + with open(github_step_summary, "a", encoding="utf-8") as wfh: + wfh.write("Core tests chosen by `test:core` label.\n") + testrun["selected_tests"]["core"] = True + if "test:no-fast" in labels: + with open(github_step_summary, "a", encoding="utf-8") as wfh: + wfh.write("Fast tests deselected by `test:no-fast` label.\n") + testrun["selected_tests"]["fast"] = False + if "test:flaky-jail" in labels: + with open(github_step_summary, "a", encoding="utf-8") as wfh: + wfh.write("Flaky jailed tests chosen by `test:flaky-jail` label.\n") + testrun["selected_tests"]["flaky"] = True if selected_changed_files: with open(github_step_summary, "a", encoding="utf-8") as wfh: wfh.write( From 2d824c91534991fafcae051afc376d7ef9e8be5c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 27 Jun 2023 16:44:44 +0100 Subject: [PATCH 42/53] Enable code coverage on pull-requests by setting the `test:coverage` label Signed-off-by: Pedro Algarvio --- .github/workflows/ci.yml | 86 ++++++++++---------- .github/workflows/templates/layout.yml.jinja | 2 +- changelog/64547.added.md | 1 + tools/ci.py | 46 ++++++----- 4 files changed, 72 insertions(+), 63 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baf729efce6..05284621f9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -513,7 +513,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: rpm cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} centos-7-pkg-tests: @@ -530,7 +530,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: rpm cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} centosstream-8-pkg-tests: @@ -547,7 +547,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: rpm cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} centosstream-9-pkg-tests: @@ -564,7 +564,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: rpm cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} debian-10-pkg-tests: @@ -581,7 +581,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: deb cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} debian-11-pkg-tests: @@ -598,7 +598,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: deb cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} debian-11-arm64-pkg-tests: @@ -615,7 +615,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: deb cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} photonos-3-pkg-tests: @@ -632,7 +632,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: rpm cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} photonos-4-pkg-tests: @@ -649,7 +649,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: rpm cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} ubuntu-2004-pkg-tests: @@ -666,7 +666,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: deb cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} ubuntu-2004-arm64-pkg-tests: @@ -683,7 +683,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: deb cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} ubuntu-2204-pkg-tests: @@ -700,7 +700,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: deb cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} ubuntu-2204-arm64-pkg-tests: @@ -717,7 +717,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: deb cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} macos-12-pkg-tests: @@ -734,7 +734,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: macos cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} windows-2016-nsis-pkg-tests: @@ -751,7 +751,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: NSIS cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} windows-2016-msi-pkg-tests: @@ -768,7 +768,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: MSI cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} windows-2019-nsis-pkg-tests: @@ -785,7 +785,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: NSIS cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} windows-2019-msi-pkg-tests: @@ -802,7 +802,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: MSI cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} windows-2022-nsis-pkg-tests: @@ -819,7 +819,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: NSIS cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} windows-2022-msi-pkg-tests: @@ -836,7 +836,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" pkg-type: MSI cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} windows-2016: @@ -854,7 +854,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} windows-2019: @@ -872,7 +872,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} windows-2022: @@ -890,7 +890,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} macos-12: @@ -908,7 +908,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} almalinux-8: @@ -926,7 +926,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} almalinux-9: @@ -944,7 +944,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} amazonlinux-2: @@ -962,7 +962,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} archlinux-lts: @@ -980,7 +980,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} centos-7: @@ -998,7 +998,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} centosstream-8: @@ -1016,7 +1016,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} centosstream-9: @@ -1034,7 +1034,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} debian-10: @@ -1052,7 +1052,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} debian-11: @@ -1070,7 +1070,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} debian-11-arm64: @@ -1088,7 +1088,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} fedora-37: @@ -1106,7 +1106,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} fedora-38: @@ -1124,7 +1124,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} opensuse-15: @@ -1142,7 +1142,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} photonos-3: @@ -1160,7 +1160,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} photonos-4: @@ -1178,7 +1178,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} ubuntu-2004: @@ -1196,7 +1196,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} ubuntu-2004-arm64: @@ -1214,7 +1214,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} ubuntu-2204: @@ -1232,7 +1232,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} ubuntu-2204-arm64: @@ -1250,7 +1250,7 @@ jobs: testrun: ${{ needs.prepare-workflow.outputs.testrun }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.11 - skip-code-coverage: ${{ github.event_name == 'pull_request' }} + skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} set-pipeline-exit-status: diff --git a/.github/workflows/templates/layout.yml.jinja b/.github/workflows/templates/layout.yml.jinja index 9ca3bf76fa8..ea52a8df37a 100644 --- a/.github/workflows/templates/layout.yml.jinja +++ b/.github/workflows/templates/layout.yml.jinja @@ -5,7 +5,7 @@ <%- set prepare_workflow_skip_pkg_test_suite = prepare_workflow_skip_pkg_test_suite|default("") %> <%- set prepare_workflow_skip_pkg_download_test_suite = prepare_workflow_skip_pkg_download_test_suite|default("") %> <%- set prepare_workflow_salt_version_input = prepare_workflow_salt_version_input|default("") %> -<%- set skip_test_coverage_check = skip_test_coverage_check|default("${{ github.event_name == 'pull_request' }}") %> +<%- set skip_test_coverage_check = skip_test_coverage_check|default("${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }}") %> <%- set skip_junit_reports_check = skip_junit_reports_check|default("${{ github.event_name == 'pull_request' }}") %> <%- set gpg_key_id = "64CBBC8173D76B3F" %> <%- set prepare_actual_release = prepare_actual_release | default(False) %> diff --git a/changelog/64547.added.md b/changelog/64547.added.md index f1bc6e6a131..e3ae3a3a449 100644 --- a/changelog/64547.added.md +++ b/changelog/64547.added.md @@ -5,3 +5,4 @@ Several fixes to the CI workflow: * Switch to using `tools` and re-use the event payload available instead of querying the GH API again to get the pull request labels * Concentrate test selection by labels to a single place +* Enable code coverage on pull-requests by setting the `test:coverage` label diff --git a/tools/ci.py b/tools/ci.py index c3f79aab355..60df792d25fb 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -389,6 +389,7 @@ def define_jobs( class TestRun(TypedDict): type: str + skip_code_coverage: bool from_filenames: NotRequired[str] selected_tests: NotRequired[dict[str, bool]] @@ -427,10 +428,31 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): if TYPE_CHECKING: assert github_step_summary is not None + labels: list[str] = [] + gh_event_path = os.environ.get("GITHUB_EVENT_PATH") or None + if gh_event_path is not None: + try: + gh_event = json.loads(open(gh_event_path).read()) + except Exception as exc: + ctx.error( + f"Could not load the GH Event payload from {gh_event_path!r}:\n", exc + ) + ctx.exit(1) + + labels.extend( + label[0] for label in _get_pr_test_labels_from_event_payload(gh_event) + ) + + skip_code_coverage = True + if "test:coverage" in labels: + skip_code_coverage = False + elif event_name != "pull_request": + skip_code_coverage = False + if event_name != "pull_request": # In this case, a full test run is in order ctx.info("Writing 'testrun' to the github outputs file") - testrun = TestRun(type="full") + testrun = TestRun(type="full", skip_code_coverage=skip_code_coverage) with open(github_output, "a", encoding="utf-8") as wfh: wfh.write(f"testrun={json.dumps(testrun)}\n") @@ -451,21 +473,6 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): ctx.error(f"Could not load the changed files from '{changed_files}': {exc}") ctx.exit(1) - labels: list[str] = [] - gh_event_path = os.environ.get("GITHUB_EVENT_PATH") or None - if gh_event_path is not None: - try: - gh_event = json.loads(open(gh_event_path).read()) - except Exception as exc: - ctx.error( - f"Could not load the GH Event payload from {gh_event_path!r}:\n", exc - ) - ctx.exit(1) - - labels.extend( - label[0] for label in _get_pr_test_labels_from_event_payload(gh_event) - ) - # So, it's a pull request... # Based on which files changed, or other things like PR labels we can # decide what to run, or even if the full test run should be running on the @@ -482,7 +489,7 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): "Full test run chosen because there was a change made " "to `cicd/golden-images.json`.\n" ) - testrun = TestRun(type="full") + testrun = TestRun(type="full", skip_code_coverage=skip_code_coverage) elif changed_pkg_requirements_files or changed_test_requirements_files: with open(github_step_summary, "a", encoding="utf-8") as wfh: wfh.write( @@ -497,15 +504,16 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): ): wfh.write(f"{path}\n") wfh.write("\n\n") - testrun = TestRun(type="full") + testrun = TestRun(type="full", skip_code_coverage=skip_code_coverage) elif "test:full" in labels: with open(github_step_summary, "a", encoding="utf-8") as wfh: wfh.write("Full test run chosen because the label `test:full` is set.\n") - testrun = TestRun(type="full") + testrun = TestRun(type="full", skip_code_coverage=skip_code_coverage) else: testrun_changed_files_path = tools.utils.REPO_ROOT / "testrun-changed-files.txt" testrun = TestRun( type="changed", + skip_code_coverage=skip_code_coverage, from_filenames=str( testrun_changed_files_path.relative_to(tools.utils.REPO_ROOT) ), From 2cf3094331d129fd9167506e8d561dc36a789594 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 22 Feb 2023 16:08:35 +0000 Subject: [PATCH 43/53] Stop using the deprecated `cgi` module. Signed-off-by: Pedro Algarvio --- salt/utils/http.py | 48 ++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/salt/utils/http.py b/salt/utils/http.py index 91c5cbf08ed..26f2e85c2ee 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -5,7 +5,7 @@ and the like, but also useful for basic HTTP testing. .. versionadded:: 2015.5.0 """ -import cgi +import email.message import gzip import http.client import http.cookiejar @@ -84,7 +84,7 @@ except ImportError: HAS_CERTIFI = False log = logging.getLogger(__name__) -USERAGENT = "Salt/{}".format(salt.version.__version__) +USERAGENT = f"Salt/{salt.version.__version__}" def __decompressContent(coding, pgctnt): @@ -170,7 +170,7 @@ def query( formdata_fieldname=None, formdata_filename=None, decode_body=True, - **kwargs + **kwargs, ): """ Query a resource, and decode the return data @@ -295,7 +295,7 @@ def query( auth = (username, password) if agent == USERAGENT: - agent = "{} http.query()".format(agent) + agent = f"{agent} http.query()" header_dict["User-agent"] = agent if backend == "requests": @@ -360,14 +360,14 @@ def query( url, params=params, files={formdata_fieldname: (formdata_filename, io.StringIO(data))}, - **req_kwargs + **req_kwargs, ) else: result = sess.request(method, url, params=params, data=data, **req_kwargs) result.raise_for_status() if stream is True: # fake a HTTP response header - header_callback("HTTP/1.0 {} MESSAGE".format(result.status_code)) + header_callback(f"HTTP/1.0 {result.status_code} MESSAGE") # fake streaming the content streaming_callback(result.content) return { @@ -483,15 +483,12 @@ def query( result_headers = dict(result.info()) result_text = result.read() if "Content-Type" in result_headers: - res_content_type, res_params = cgi.parse_header( - result_headers["Content-Type"] - ) - if ( - res_content_type.startswith("text/") - and "charset" in res_params - and not isinstance(result_text, str) - ): - result_text = result_text.decode(res_params["charset"]) + msg = email.message.EmailMessage() + msg.add_header("Content-Type", result_headers["Content-Type"]) + if msg.get_content_type().startswith("text/"): + content_charset = msg.get_content_charset() + if content_charset and not isinstance(result_text, str): + result_text = result_text.decode(content_charset) if isinstance(result_text, bytes) and decode_body: result_text = result_text.decode("utf-8") ret["body"] = result_text @@ -636,15 +633,12 @@ def query( result_headers = result.headers result_text = result.body if "Content-Type" in result_headers: - res_content_type, res_params = cgi.parse_header( - result_headers["Content-Type"] - ) - if ( - res_content_type.startswith("text/") - and "charset" in res_params - and not isinstance(result_text, str) - ): - result_text = result_text.decode(res_params["charset"]) + msg = email.message.EmailMessage() + msg.add_header("Content-Type", result_headers["Content-Type"]) + if msg.get_content_type().startswith("text/"): + content_charset = msg.get_content_charset() + if content_charset and not isinstance(result_text, str): + result_text = result_text.decode(content_charset) if isinstance(result_text, bytes) and decode_body: result_text = result_text.decode("utf-8") ret["body"] = result_text @@ -1038,12 +1032,12 @@ def _sanitize_url_components(comp_list, field): """ if not comp_list: return "" - elif comp_list[0].startswith("{}=".format(field)): - ret = "{}=XXXXXXXXXX&".format(field) + elif comp_list[0].startswith(f"{field}="): + ret = f"{field}=XXXXXXXXXX&" comp_list.remove(comp_list[0]) return ret + _sanitize_url_components(comp_list, field) else: - ret = "{}&".format(comp_list[0]) + ret = f"{comp_list[0]}&" comp_list.remove(comp_list[0]) return ret + _sanitize_url_components(comp_list, field) From 1f9d094b8f957ff374d5adfbf15cf6cce8277c7c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 22 Feb 2023 16:12:16 +0000 Subject: [PATCH 44/53] Stop using the deprecated `pipes` module Signed-off-by: Pedro Algarvio --- salt/modules/container_resource.py | 74 +++----- salt/modules/deb_postgres.py | 16 +- salt/modules/dockermod.py | 178 ++++++++---------- salt/modules/lxc.py | 278 +++++++++++++-------------- salt/modules/mac_keychain.py | 32 +--- salt/modules/macpackage.py | 45 ++--- salt/modules/openstack_config.py | 41 ++-- salt/modules/postgres.py | 116 ++++++------ salt/utils/cloud.py | 289 +++++++++++++---------------- salt/utils/jinja.py | 25 ++- 10 files changed, 467 insertions(+), 627 deletions(-) diff --git a/salt/modules/container_resource.py b/salt/modules/container_resource.py index a29cba2e468..ceec72a7b20 100644 --- a/salt/modules/container_resource.py +++ b/salt/modules/container_resource.py @@ -8,13 +8,11 @@ These functions are not designed to be called directly, but instead from the :mod:`docker ` execution modules. They provide for common logic to be re-used for common actions. """ - - import copy import functools import logging import os -import pipes +import shlex import time import traceback @@ -68,14 +66,14 @@ def _nsenter(pid): """ Return the nsenter command to attach to the named container """ - return "nsenter --target {} --mount --uts --ipc --net --pid".format(pid) + return f"nsenter --target {pid} --mount --uts --ipc --net --pid" def _get_md5(name, path, run_func): """ Get the MD5 checksum of a file from a container """ - output = run_func(name, "md5sum {}".format(pipes.quote(path)), ignore_retcode=True)[ + output = run_func(name, f"md5sum {shlex.quote(path)}", ignore_retcode=True)[ "stdout" ] try: @@ -102,10 +100,10 @@ def cache_file(source): if source.startswith("salt://"): cached_source = __salt__["cp.cache_file"](source) if not cached_source: - raise CommandExecutionError("Unable to cache {}".format(source)) + raise CommandExecutionError(f"Unable to cache {source}") return cached_source except AttributeError: - raise SaltInvocationError("Invalid source file {}".format(source)) + raise SaltInvocationError(f"Invalid source file {source}") return source @@ -164,55 +162,47 @@ def run( if exec_driver == "lxc-attach": full_cmd = "lxc-attach " if path: - full_cmd += "-P {} ".format(pipes.quote(path)) + full_cmd += f"-P {shlex.quote(path)} " if keep_env is not True: full_cmd += "--clear-env " if "PATH" not in to_keep: - full_cmd += "--set-var {} ".format(PATH) + full_cmd += f"--set-var {PATH} " # --clear-env results in a very restrictive PATH # (/bin:/usr/bin), use a good fallback. full_cmd += " ".join( [ - "--set-var {}={}".format(x, pipes.quote(os.environ[x])) + f"--set-var {x}={shlex.quote(os.environ[x])}" for x in to_keep if x in os.environ ] ) - full_cmd += " -n {} -- {}".format(pipes.quote(name), cmd) + full_cmd += f" -n {shlex.quote(name)} -- {cmd}" elif exec_driver == "nsenter": - pid = __salt__["{}.pid".format(container_type)](name) - full_cmd = "nsenter --target {} --mount --uts --ipc --net --pid -- ".format(pid) + pid = __salt__[f"{container_type}.pid"](name) + full_cmd = f"nsenter --target {pid} --mount --uts --ipc --net --pid -- " if keep_env is not True: full_cmd += "env -i " if "PATH" not in to_keep: - full_cmd += "{} ".format(PATH) + full_cmd += f"{PATH} " full_cmd += " ".join( - [ - "{}={}".format(x, pipes.quote(os.environ[x])) - for x in to_keep - if x in os.environ - ] + [f"{x}={shlex.quote(os.environ[x])}" for x in to_keep if x in os.environ] ) - full_cmd += " {}".format(cmd) + full_cmd += f" {cmd}" elif exec_driver == "docker-exec": # We're using docker exec on the CLI as opposed to via docker-py, since # the Docker API doesn't return stdout and stderr separately. full_cmd = "docker exec " if stdin: full_cmd += "-i " - full_cmd += "{} ".format(name) + full_cmd += f"{name} " if keep_env is not True: full_cmd += "env -i " if "PATH" not in to_keep: - full_cmd += "{} ".format(PATH) + full_cmd += f"{PATH} " full_cmd += " ".join( - [ - "{}={}".format(x, pipes.quote(os.environ[x])) - for x in to_keep - if x in os.environ - ] + [f"{x}={shlex.quote(os.environ[x])}" for x in to_keep if x in os.environ] ) - full_cmd += " {}".format(cmd) + full_cmd += f" {cmd}" if not use_vt: ret = __salt__[cmd_func]( @@ -299,13 +289,13 @@ def copy_to( salt myminion container_resource.copy_to mycontainer /local/file/path /container/file/path container_type=docker exec_driver=nsenter """ # Get the appropriate functions - state = __salt__["{}.state".format(container_type)] + state = __salt__[f"{container_type}.state"] def run_all(*args, **akwargs): akwargs = copy.deepcopy(akwargs) if container_type in ["lxc"] and "path" not in akwargs: akwargs["path"] = path - return __salt__["{}.run_all".format(container_type)](*args, **akwargs) + return __salt__[f"{container_type}.run_all"](*args, **akwargs) state_kwargs = {} cmd_kwargs = {"ignore_retcode": True} @@ -321,7 +311,7 @@ def copy_to( c_state = _state(name) if c_state != "running": - raise CommandExecutionError("Container '{}' is not running".format(name)) + raise CommandExecutionError(f"Container '{name}' is not running") local_file = cache_file(source) source_dir, source_name = os.path.split(local_file) @@ -330,17 +320,14 @@ def copy_to( if not os.path.isabs(local_file): raise SaltInvocationError("Source path must be absolute") elif not os.path.exists(local_file): - raise SaltInvocationError("Source file {} does not exist".format(local_file)) + raise SaltInvocationError(f"Source file {local_file} does not exist") elif not os.path.isfile(local_file): raise SaltInvocationError("Source must be a regular file") # Destination file sanity checks if not os.path.isabs(dest): raise SaltInvocationError("Destination path must be absolute") - if ( - run_all(name, "test -d {}".format(pipes.quote(dest)), **cmd_kwargs)["retcode"] - == 0 - ): + if run_all(name, f"test -d {shlex.quote(dest)}", **cmd_kwargs)["retcode"] == 0: # Destination is a directory, full path to dest file will include the # basename of the source file. dest = os.path.join(dest, source_name) @@ -350,14 +337,12 @@ def copy_to( # parent directory. dest_dir, dest_name = os.path.split(dest) if ( - run_all(name, "test -d {}".format(pipes.quote(dest_dir)), **cmd_kwargs)[ - "retcode" - ] + run_all(name, f"test -d {shlex.quote(dest_dir)}", **cmd_kwargs)["retcode"] != 0 ): if makedirs: result = run_all( - name, "mkdir -p {}".format(pipes.quote(dest_dir)), **cmd_kwargs + name, f"mkdir -p {shlex.quote(dest_dir)}", **cmd_kwargs ) if result["retcode"] != 0: error = ( @@ -375,10 +360,7 @@ def copy_to( ) if ( not overwrite - and run_all(name, "test -e {}".format(pipes.quote(dest)), **cmd_kwargs)[ - "retcode" - ] - == 0 + and run_all(name, f"test -e {shlex.quote(dest)}", **cmd_kwargs)["retcode"] == 0 ): raise CommandExecutionError( "Destination path {} already exists. Use overwrite=True to " @@ -401,14 +383,14 @@ def copy_to( if exec_driver == "lxc-attach": lxcattach = "lxc-attach" if path: - lxcattach += " -P {}".format(pipes.quote(path)) + lxcattach += f" -P {shlex.quote(path)}" copy_cmd = ( 'cat "{0}" | {4} --clear-env --set-var {1} -n {2} -- tee "{3}"'.format( local_file, PATH, name, dest, lxcattach ) ) elif exec_driver == "nsenter": - pid = __salt__["{}.pid".format(container_type)](name) + pid = __salt__[f"{container_type}.pid"](name) copy_cmd = 'cat "{}" | {} env -i {} tee "{}"'.format( local_file, _nsenter(pid), PATH, dest ) diff --git a/salt/modules/deb_postgres.py b/salt/modules/deb_postgres.py index 3ecd4a8ba49..d92859562d4 100644 --- a/salt/modules/deb_postgres.py +++ b/salt/modules/deb_postgres.py @@ -2,10 +2,8 @@ Module to provide Postgres compatibility to salt for debian family specific tools. """ - - import logging -import pipes +import shlex import salt.utils.path @@ -76,7 +74,7 @@ def cluster_create( cmd += ["--data-checksums"] if wal_segsize: cmd += ["--wal-segsize", wal_segsize] - cmdstr = " ".join([pipes.quote(c) for c in cmd]) + cmdstr = " ".join([shlex.quote(c) for c in cmd]) ret = __salt__["cmd.run_all"](cmdstr, python_shell=False) if ret.get("retcode", 0) != 0: log.error("Error creating a Postgresql cluster %s/%s", version, name) @@ -97,7 +95,7 @@ def cluster_list(verbose=False): salt '*' postgres.cluster_list verbose=True """ cmd = [salt.utils.path.which("pg_lsclusters"), "--no-header"] - ret = __salt__["cmd.run_all"](" ".join([pipes.quote(c) for c in cmd])) + ret = __salt__["cmd.run_all"](" ".join([shlex.quote(c) for c in cmd])) if ret.get("retcode", 0) != 0: log.error("Error listing clusters") cluster_dict = _parse_pg_lscluster(ret["stdout"]) @@ -118,7 +116,7 @@ def cluster_exists(version, name="main"): salt '*' postgres.cluster_exists '9.3' 'main' """ - return "{}/{}".format(version, name) in cluster_list() + return f"{version}/{name}" in cluster_list() def cluster_remove(version, name="main", stop=False): @@ -141,13 +139,13 @@ def cluster_remove(version, name="main", stop=False): if stop: cmd += ["--stop"] cmd += [str(version), name] - cmdstr = " ".join([pipes.quote(c) for c in cmd]) + cmdstr = " ".join([shlex.quote(c) for c in cmd]) ret = __salt__["cmd.run_all"](cmdstr, python_shell=False) # FIXME - return Boolean ? if ret.get("retcode", 0) != 0: log.error("Error removing a Postgresql cluster %s/%s", version, name) else: - ret["changes"] = "Successfully removed cluster {}/{}".format(version, name) + ret["changes"] = f"Successfully removed cluster {version}/{name}" return ret @@ -158,7 +156,7 @@ def _parse_pg_lscluster(output): cluster_dict = {} for line in output.splitlines(): version, name, port, status, user, datadir, log = line.split() - cluster_dict["{}/{}".format(version, name)] = { + cluster_dict[f"{version}/{name}"] = { "port": int(port), "status": status, "user": user, diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 3e35700788d..b58fd1b32bc 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -204,8 +204,8 @@ import gzip import json import logging import os -import pipes import re +import shlex import shutil import string import subprocess @@ -252,7 +252,6 @@ except ImportError: HAS_NSENTER = bool(salt.utils.path.which("nsenter")) -# Set up logging log = logging.getLogger(__name__) # Don't shadow built-in's. @@ -392,7 +391,7 @@ def _get_client(timeout=NOTSET, **kwargs): ) except Exception as exc: # pylint: disable=broad-except raise CommandExecutionError( - "Docker machine {} failed: {}".format(docker_machine, exc) + f"Docker machine {docker_machine} failed: {exc}" ) try: # docker-py 2.0 renamed this client attribute @@ -492,7 +491,7 @@ def _change_state(name, action, expected, *args, **kwargs): return { "result": False, "state": {"old": expected, "new": expected}, - "comment": "Container '{}' already {}".format(name, expected), + "comment": f"Container '{name}' already {expected}", } _client_wrapper(action, name, *args, **kwargs) _clear_context() @@ -530,9 +529,7 @@ def _get_md5(name, path): """ Get the MD5 checksum of a file from a container """ - output = run_stdout( - name, "md5sum {}".format(pipes.quote(path)), ignore_retcode=True - ) + output = run_stdout(name, f"md5sum {shlex.quote(path)}", ignore_retcode=True) try: return output.split()[0] except IndexError: @@ -611,7 +608,7 @@ def _scrub_links(links, name): if isinstance(links, list): ret = [] for l in links: - ret.append(l.replace("/{}/".format(name), "/", 1)) + ret.append(l.replace(f"/{name}/", "/", 1)) else: ret = links @@ -634,11 +631,11 @@ def _size_fmt(num): try: num = int(num) if num < 1024: - return "{} bytes".format(num) + return f"{num} bytes" num /= 1024.0 for unit in ("KiB", "MiB", "GiB", "TiB", "PiB"): if num < 1024.0: - return "{:3.1f} {}".format(num, unit) + return f"{num:3.1f} {unit}" num /= 1024.0 except Exception: # pylint: disable=broad-except log.error("Unable to format file size for '%s'", num) @@ -653,7 +650,7 @@ def _client_wrapper(attr, *args, **kwargs): catch_api_errors = kwargs.pop("catch_api_errors", True) func = getattr(__context__["docker.client"], attr, None) if func is None or not hasattr(func, "__call__"): - raise SaltInvocationError("Invalid client action '{}'".format(attr)) + raise SaltInvocationError(f"Invalid client action '{attr}'") if attr in ("push", "pull"): try: # Refresh auth config from config.json @@ -673,7 +670,7 @@ def _client_wrapper(attr, *args, **kwargs): if catch_api_errors: # Generic handling of Docker API errors raise CommandExecutionError( - "Error {}: {}".format(exc.response.status_code, exc.explanation) + f"Error {exc.response.status_code}: {exc.explanation}" ) else: # Allow API errors to be caught further up the stack @@ -688,9 +685,9 @@ def _client_wrapper(attr, *args, **kwargs): # If we're here, it's because an exception was caught earlier, and the # API command failed. - msg = "Unable to perform {}".format(attr) + msg = f"Unable to perform {attr}" if err: - msg += ": {}".format(err) + msg += f": {err}" raise CommandExecutionError(msg) @@ -717,7 +714,7 @@ def _import_status(data, item, repo_name, repo_tag): return elif all(x in string.hexdigits for x in status): # Status is an image ID - data["Image"] = "{}:{}".format(repo_name, repo_tag) + data["Image"] = f"{repo_name}:{repo_tag}" data["Id"] = status except (AttributeError, TypeError): pass @@ -876,7 +873,7 @@ def _get_create_kwargs( ignore_collisions=False, validate_ip_addrs=True, client_args=None, - **kwargs + **kwargs, ): """ Take input kwargs and return a kwargs dict to pass to docker-py's @@ -894,7 +891,7 @@ def _get_create_kwargs( skip_translate=skip_translate, ignore_collisions=ignore_collisions, validate_ip_addrs=validate_ip_addrs, - **__utils__["args.clean_kwargs"](**kwargs) + **__utils__["args.clean_kwargs"](**kwargs), ) if networks: @@ -907,7 +904,7 @@ def _get_create_kwargs( log.error( "docker.create: Error getting client args: '%s'", exc, exc_info=True ) - raise CommandExecutionError("Failed to get client args: {}".format(exc)) + raise CommandExecutionError(f"Failed to get client args: {exc}") full_host_config = {} host_kwargs = {} @@ -1468,15 +1465,15 @@ def login(*registries): results = ret.setdefault("Results", {}) for registry in registries: if registry not in registry_auth: - errors.append("No match found for registry '{}'".format(registry)) + errors.append(f"No match found for registry '{registry}'") continue try: username = registry_auth[registry]["username"] password = registry_auth[registry]["password"] except TypeError: - errors.append("Invalid configuration for registry '{}'".format(registry)) + errors.append(f"Invalid configuration for registry '{registry}'") except KeyError as exc: - errors.append("Missing {} for registry '{}'".format(exc, registry)) + errors.append(f"Missing {exc} for registry '{registry}'") else: cmd = ["docker", "login", "-u", username, "-p", password] if registry.lower() != "hub": @@ -1562,7 +1559,7 @@ def logout(*registries): results = ret.setdefault("Results", {}) for registry in registries: if registry not in registry_auth: - errors.append("No match found for registry '{}'".format(registry)) + errors.append(f"No match found for registry '{registry}'") continue else: cmd = ["docker", "logout"] @@ -1684,7 +1681,7 @@ def exists(name): salt myminion docker.exists mycontainer """ - contextkey = "docker.exists.{}".format(name) + contextkey = f"docker.exists.{name}" if contextkey in __context__: return __context__[contextkey] try: @@ -1775,7 +1772,7 @@ def history(name, quiet=False): ) for param in ("Size",): if param in step: - step["{}_Human".format(param)] = _size_fmt(step[param]) + step[f"{param}_Human"] = _size_fmt(step[param]) ret.append(copy.deepcopy(step)) if quiet: return [x.get("Command") for x in ret] @@ -1837,9 +1834,7 @@ def images(verbose=False, **kwargs): ) for param in ("Size", "VirtualSize"): if param in bucket.get(img_id, {}): - bucket[img_id]["{}_Human".format(param)] = _size_fmt( - bucket[img_id][param] - ) + bucket[img_id][f"{param}_Human"] = _size_fmt(bucket[img_id][param]) context_data = __context__.get("docker.images", {}) ret = copy.deepcopy(context_data.get("tagged", {})) @@ -1922,7 +1917,7 @@ def inspect(name): raise raise CommandExecutionError( - "Error 404: No such image/container/volume/network: {}".format(name) + f"Error 404: No such image/container/volume/network: {name}" ) @@ -1978,7 +1973,7 @@ def inspect_image(name): ret = _client_wrapper("inspect_image", name) for param in ("Size", "VirtualSize"): if param in ret: - ret["{}_Human".format(param)] = _size_fmt(ret[param]) + ret[f"{param}_Human"] = _size_fmt(ret[param]) return ret @@ -2272,7 +2267,7 @@ def port(name, private_port=None): else: # Sanity checks if isinstance(private_port, int): - pattern = "{}/*".format(private_port) + pattern = f"{private_port}/*" else: err = ( "Invalid private_port '{}'. Must either be a port number, " @@ -2393,7 +2388,7 @@ def state(name): salt myminion docker.state mycontainer """ - contextkey = "docker.state.{}".format(name) + contextkey = f"docker.state.{name}" if contextkey in __context__: return __context__[contextkey] __context__[contextkey] = _get_state(inspect_container(name)) @@ -2433,9 +2428,7 @@ def search(name, official=False, trusted=False): """ response = _client_wrapper("search", name) if not response: - raise CommandExecutionError( - "No images matched the search string '{}'".format(name) - ) + raise CommandExecutionError(f"No images matched the search string '{name}'") key_map = { "description": "Description", @@ -2550,7 +2543,7 @@ def create( ignore_collisions=False, validate_ip_addrs=True, client_timeout=salt.utils.dockermod.CLIENT_TIMEOUT, - **kwargs + **kwargs, ): """ Create a new container @@ -3276,7 +3269,7 @@ def create( skip_translate=skip_translate, ignore_collisions=ignore_collisions, validate_ip_addrs=validate_ip_addrs, - **kwargs + **kwargs, ) if unused_kwargs: @@ -3288,7 +3281,7 @@ def create( log.debug( "docker.create: creating container %susing the following arguments: %s", - "with name '{}' ".format(name) if name is not None else "", + f"with name '{name}' " if name is not None else "", kwargs, ) time_started = time.time() @@ -3326,7 +3319,7 @@ def run_container( replace=False, force=False, networks=None, - **kwargs + **kwargs, ): """ .. versionadded:: 2018.3.0 @@ -3428,7 +3421,7 @@ def run_container( skip_translate=skip_translate, ignore_collisions=ignore_collisions, validate_ip_addrs=validate_ip_addrs, - **kwargs + **kwargs, ) # _get_create_kwargs() will have processed auto_remove and put it into the @@ -3453,7 +3446,7 @@ def run_container( log.debug( "docker.create: creating container %susing the following arguments: %s", - "with name '{}' ".format(name) if name is not None else "", + f"with name '{name}' " if name is not None else "", kwargs, ) @@ -3493,7 +3486,7 @@ def run_container( rm_(name) except CommandExecutionError as rm_exc: exc_info.setdefault("other_errors", []).append( - "Failed to auto_remove container: {}".format(rm_exc) + f"Failed to auto_remove container: {rm_exc}" ) # Raise original exception with additional info raise CommandExecutionError(exc.__str__(), info=exc_info) @@ -3588,7 +3581,7 @@ def copy_from(name, source, dest, overwrite=False, makedirs=False): """ c_state = state(name) if c_state != "running": - raise CommandExecutionError("Container '{}' is not running".format(name)) + raise CommandExecutionError(f"Container '{name}' is not running") # Destination file sanity checks if not os.path.isabs(dest): @@ -3614,9 +3607,7 @@ def copy_from(name, source, dest, overwrite=False, makedirs=False): ) ) else: - raise SaltInvocationError( - "Directory {} does not exist".format(dest_dir) - ) + raise SaltInvocationError(f"Directory {dest_dir} does not exist") if not overwrite and os.path.exists(dest): raise CommandExecutionError( "Destination path {} already exists. Use overwrite=True to " @@ -3627,19 +3618,14 @@ def copy_from(name, source, dest, overwrite=False, makedirs=False): if not os.path.isabs(source): raise SaltInvocationError("Source path must be absolute") else: - if ( - retcode(name, "test -e {}".format(pipes.quote(source)), ignore_retcode=True) - == 0 - ): + if retcode(name, f"test -e {shlex.quote(source)}", ignore_retcode=True) == 0: if ( - retcode( - name, "test -f {}".format(pipes.quote(source)), ignore_retcode=True - ) + retcode(name, f"test -f {shlex.quote(source)}", ignore_retcode=True) != 0 ): raise SaltInvocationError("Source must be a regular file") else: - raise SaltInvocationError("Source file {} does not exist".format(source)) + raise SaltInvocationError(f"Source file {source} does not exist") # Before we try to replace the file, compare checksums. source_md5 = _get_md5(name, source) @@ -3652,7 +3638,7 @@ def copy_from(name, source, dest, overwrite=False, makedirs=False): try: src_path = ":".join((name, source)) except TypeError: - src_path = "{}:{}".format(name, source) + src_path = f"{name}:{source}" cmd = ["docker", "cp", src_path, dest_dir] __salt__["cmd.run"](cmd, python_shell=False) return source_md5 == __salt__["file.get_sum"](dest, "md5") @@ -3779,7 +3765,7 @@ def export(name, path, overwrite=False, makedirs=False, compression=None, **kwar salt myminion docker.export mycontainer /tmp/mycontainer.tar salt myminion docker.export mycontainer /tmp/mycontainer.tar.xz push=True """ - err = "Path '{}' is not absolute".format(path) + err = f"Path '{path}' is not absolute" try: if not os.path.isabs(path): raise SaltInvocationError(err) @@ -3787,7 +3773,7 @@ def export(name, path, overwrite=False, makedirs=False, compression=None, **kwar raise SaltInvocationError(err) if os.path.exists(path) and not overwrite: - raise CommandExecutionError("{} already exists".format(path)) + raise CommandExecutionError(f"{path} already exists") if compression is None: if path.endswith(".tar.gz") or path.endswith(".tgz"): @@ -3810,7 +3796,7 @@ def export(name, path, overwrite=False, makedirs=False, compression=None, **kwar compression = "xz" if compression and compression not in ("gzip", "bzip2", "xz"): - raise SaltInvocationError("Invalid compression type '{}'".format(compression)) + raise SaltInvocationError(f"Invalid compression type '{compression}'") parent_dir = os.path.dirname(path) if not os.path.isdir(parent_dir): @@ -3823,16 +3809,14 @@ def export(name, path, overwrite=False, makedirs=False, compression=None, **kwar os.makedirs(parent_dir) except OSError as exc: raise CommandExecutionError( - "Unable to make parent dir {}: {}".format(parent_dir, exc) + f"Unable to make parent dir {parent_dir}: {exc}" ) if compression == "gzip": try: out = gzip.open(path, "wb") except OSError as exc: - raise CommandExecutionError( - "Unable to open {} for writing: {}".format(path, exc) - ) + raise CommandExecutionError(f"Unable to open {path} for writing: {exc}") elif compression == "bzip2": compressor = bz2.BZ2Compressor() elif compression == "xz": @@ -3870,9 +3854,7 @@ def export(name, path, overwrite=False, makedirs=False, compression=None, **kwar os.remove(path) except OSError: pass - raise CommandExecutionError( - "Error occurred during container export: {}".format(exc) - ) + raise CommandExecutionError(f"Error occurred during container export: {exc}") finally: out.close() ret = {"Time_Elapsed": time.time() - time_started} @@ -4103,7 +4085,7 @@ def build( # For the build function in the low-level API, the "tag" refers to the full # tag (e.g. myuser/myimage:mytag). This is different than in other # functions, where the repo and tag are passed separately. - image_tag = "{}:{}".format(repository, tag) if repository and tag else None + image_tag = f"{repository}:{tag}" if repository and tag else None time_started = time.time() response = _client_wrapper( @@ -4122,7 +4104,7 @@ def build( if not response: raise CommandExecutionError( - "Build failed for {}, no response returned from Docker API".format(path) + f"Build failed for {path}, no response returned from Docker API" ) stream_data = [] @@ -4145,7 +4127,7 @@ def build( if "Id" not in ret: # API returned information, but there was no confirmation of a # successful build. - msg = "Build failed for {}".format(path) + msg = f"Build failed for {path}" log.error(msg) log.error(stream_data) if errors: @@ -4156,7 +4138,7 @@ def build( if resolved_tag: ret["Image"] = resolved_tag else: - ret["Warning"] = "Failed to tag image as {}".format(image_tag) + ret["Warning"] = f"Failed to tag image as {image_tag}" if api_response: ret["API_Response"] = stream_data @@ -4363,7 +4345,7 @@ def import_(source, repository, tag="latest", api_response=False): if not response: raise CommandExecutionError( - "Import failed for {}, no response returned from Docker API".format(source) + f"Import failed for {source}, no response returned from Docker API" ) elif api_response: ret["API_Response"] = response @@ -4383,7 +4365,7 @@ def import_(source, repository, tag="latest", api_response=False): if "Id" not in ret: # API returned information, but there was no confirmation of a # successful push. - msg = "Import failed for {}".format(source) + msg = f"Import failed for {source}" if errors: msg += ". Error(s) follow:\n\n{}".format("\n\n".join(errors)) raise CommandExecutionError(msg) @@ -4458,7 +4440,7 @@ def load(path, repository=None, tag=None): local_path = __salt__["container_resource.cache_file"](path) if not os.path.isfile(local_path): - raise CommandExecutionError("Source file {} does not exist".format(path)) + raise CommandExecutionError(f"Source file {path} does not exist") pre = images(all=True) cmd = ["docker", "load", "-i", local_path] @@ -4468,7 +4450,7 @@ def load(path, repository=None, tag=None): _clear_context() post = images(all=True) if result["retcode"] != 0: - msg = "Failed to load image(s) from {}".format(path) + msg = f"Failed to load image(s) from {path}" if result["stderr"]: msg += ": {}".format(result["stderr"]) raise CommandExecutionError(msg) @@ -4489,7 +4471,7 @@ def load(path, repository=None, tag=None): # strings when passed (e.g. a numeric tag would be loaded as an int # or float), and because the tag_ function will stringify them if # need be, a str.format is the correct thing to do here. - tagged_image = "{}:{}".format(repository, tag) + tagged_image = f"{repository}:{tag}" try: result = tag_(top_level_images[0], repository=repository, tag=tag) ret["Image"] = tagged_image @@ -4526,7 +4508,7 @@ def layers(name): ): ret.append(line) if not ret: - raise CommandExecutionError("Image '{}' not found".format(name)) + raise CommandExecutionError(f"Image '{name}' not found") return ret @@ -4597,7 +4579,7 @@ def pull( if not response: raise CommandExecutionError( - "Pull failed for {}, no response returned from Docker API".format(image) + f"Pull failed for {image}, no response returned from Docker API" ) elif api_response: ret["API_Response"] = response @@ -4610,7 +4592,7 @@ def pull( event = salt.utils.json.loads(event) except Exception as exc: # pylint: disable=broad-except raise CommandExecutionError( - "Unable to interpret API event: '{}'".format(event), + f"Unable to interpret API event: '{event}'", info={"Error": exc.__str__()}, ) try: @@ -4692,7 +4674,7 @@ def push( if not response: raise CommandExecutionError( - "Push failed for {}, no response returned from Docker API".format(image) + f"Push failed for {image}, no response returned from Docker API" ) elif api_response: ret["API_Response"] = response @@ -4704,7 +4686,7 @@ def push( event = salt.utils.json.loads(event) except Exception as exc: # pylint: disable=broad-except raise CommandExecutionError( - "Unable to interpret API event: '{}'".format(event), + f"Unable to interpret API event: '{event}'", info={"Error": exc.__str__()}, ) try: @@ -4784,9 +4766,7 @@ def rmi(*names, **kwargs): err += "image(s): {}".format(", ".join(deps["Images"])) errors.append(err) else: - errors.append( - "Error {}: {}".format(exc.response.status_code, exc.explanation) - ) + errors.append(f"Error {exc.response.status_code}: {exc.explanation}") _clear_context() ret = { @@ -4874,7 +4854,7 @@ def save(name, path, overwrite=False, makedirs=False, compression=None, **kwargs salt myminion docker.save centos:7 /tmp/cent7.tar salt myminion docker.save 0123456789ab cdef01234567 /tmp/saved.tar """ - err = "Path '{}' is not absolute".format(path) + err = f"Path '{path}' is not absolute" try: if not os.path.isabs(path): raise SaltInvocationError(err) @@ -4882,7 +4862,7 @@ def save(name, path, overwrite=False, makedirs=False, compression=None, **kwargs raise SaltInvocationError(err) if os.path.exists(path) and not overwrite: - raise CommandExecutionError("{} already exists".format(path)) + raise CommandExecutionError(f"{path} already exists") if compression is None: if path.endswith(".tar.gz") or path.endswith(".tgz"): @@ -4905,7 +4885,7 @@ def save(name, path, overwrite=False, makedirs=False, compression=None, **kwargs compression = "xz" if compression and compression not in ("gzip", "bzip2", "xz"): - raise SaltInvocationError("Invalid compression type '{}'".format(compression)) + raise SaltInvocationError(f"Invalid compression type '{compression}'") parent_dir = os.path.dirname(path) if not os.path.isdir(parent_dir): @@ -4927,7 +4907,7 @@ def save(name, path, overwrite=False, makedirs=False, compression=None, **kwargs time_started = time.time() result = __salt__["cmd.run_all"](cmd, python_shell=False) if result["retcode"] != 0: - err = "Failed to save image(s) to {}".format(path) + err = f"Failed to save image(s) to {path}" if result["stderr"]: err += ": {}".format(result["stderr"]) raise CommandExecutionError(err) @@ -4937,9 +4917,7 @@ def save(name, path, overwrite=False, makedirs=False, compression=None, **kwargs try: out = gzip.open(path, "wb") except OSError as exc: - raise CommandExecutionError( - "Unable to open {} for writing: {}".format(path, exc) - ) + raise CommandExecutionError(f"Unable to open {path} for writing: {exc}") elif compression == "bzip2": compressor = bz2.BZ2Compressor() elif compression == "xz": @@ -4975,9 +4953,7 @@ def save(name, path, overwrite=False, makedirs=False, compression=None, **kwargs os.remove(path) except OSError: pass - raise CommandExecutionError( - "Error occurred during image save: {}".format(exc) - ) + raise CommandExecutionError(f"Error occurred during image save: {exc}") finally: try: # Clean up temp file @@ -5097,7 +5073,7 @@ def create_network( ignore_collisions=False, validate_ip_addrs=True, client_timeout=salt.utils.dockermod.CLIENT_TIMEOUT, - **kwargs + **kwargs, ): """ .. versionchanged:: 2018.3.0 @@ -5337,7 +5313,7 @@ def create_network( skip_translate=skip_translate, ignore_collisions=ignore_collisions, validate_ip_addrs=validate_ip_addrs, - **__utils__["args.clean_kwargs"](**kwargs) + **__utils__["args.clean_kwargs"](**kwargs), ) if "ipam" not in kwargs: @@ -5669,7 +5645,7 @@ def pause(name): return { "result": False, "state": {"old": orig_state, "new": orig_state}, - "comment": "Container '{}' is stopped, cannot pause".format(name), + "comment": f"Container '{name}' is stopped, cannot pause", } return _change_state(name, "pause", "paused") @@ -5768,7 +5744,7 @@ def start_(name): return { "result": False, "state": {"old": orig_state, "new": orig_state}, - "comment": "Container '{}' is paused, cannot start".format(name), + "comment": f"Container '{name}' is paused, cannot start", } return _change_state(name, "start", "running") @@ -5873,7 +5849,7 @@ def unpause(name): return { "result": False, "state": {"old": orig_state, "new": orig_state}, - "comment": "Container '{}' is stopped, cannot unpause".format(name), + "comment": f"Container '{name}' is stopped, cannot unpause", } return _change_state(name, "unpause", "running") @@ -5922,7 +5898,7 @@ def wait(name, ignore_already_stopped=False, fail_on_exit_status=False): # Container doesn't exist anymore return { "result": ignore_already_stopped, - "comment": "Container '{}' absent".format(name), + "comment": f"Container '{name}' absent", } already_stopped = pre == "stopped" response = _client_wrapper("wait", name) @@ -5946,7 +5922,7 @@ def wait(name, ignore_already_stopped=False, fail_on_exit_status=False): "exit_status": response, } if already_stopped: - result["comment"] = "Container '{}' already stopped".format(name) + result["comment"] = f"Container '{name}' already stopped" if fail_on_exit_status and result["result"]: result["result"] = result["exit_status"] == 0 return result @@ -5959,7 +5935,7 @@ def prune( build=False, volumes=False, system=None, - **filters + **filters, ): """ .. versionadded:: 2019.2.0 @@ -6645,7 +6621,7 @@ def script_retcode( def _generate_tmp_path(): - return os.path.join("/tmp", "salt.docker.{}".format(uuid.uuid4().hex[:6])) + return os.path.join("/tmp", f"salt.docker.{uuid.uuid4().hex[:6]}") def _prepare_trans_tar(name, sls_opts, mods=None, pillar=None, extra_filerefs=""): @@ -6780,7 +6756,7 @@ def call(name, function, *args, **kwargs): ] + list(args) + [ - "{}={}".format(key, value) + f"{key}={value}" for (key, value) in kwargs.items() if not key.startswith("__") ] diff --git a/salt/modules/lxc.py b/salt/modules/lxc.py index bea6445db98..444359bd216 100644 --- a/salt/modules/lxc.py +++ b/salt/modules/lxc.py @@ -12,9 +12,9 @@ import datetime import difflib import logging import os -import pipes import random import re +import shlex import shutil import string import tempfile @@ -119,7 +119,7 @@ def version(): ver = Version(cversion["stdout"]) if ver < Version("1.0"): raise CommandExecutionError("LXC should be at least 1.0") - __context__[k] = "{}".format(ver) + __context__[k] = f"{ver}" return __context__.get(k, None) @@ -141,7 +141,7 @@ def _ip_sort(ip): idx = "201" elif "::" in ip: idx = "100" - return "{}___{}".format(idx, ip) + return f"{idx}___{ip}" def search_lxc_bridges(): @@ -173,7 +173,7 @@ def search_lxc_bridges(): for ifc, ip in __grains__.get("ip_interfaces", {}).items(): if ifc in running_bridges: bridges.add(ifc) - elif os.path.exists("/sys/devices/virtual/net/{}/bridge".format(ifc)): + elif os.path.exists(f"/sys/devices/virtual/net/{ifc}/bridge"): bridges.add(ifc) bridges = list(bridges) # if we found interfaces that have lxc in their names @@ -186,7 +186,7 @@ def search_lxc_bridges(): pref = "a" elif "br0" == a: pref = "c" - return "{}_{}".format(pref, a) + return f"{pref}_{a}" bridges.sort(key=sort_bridges) __context__["lxc.bridges"] = bridges @@ -439,12 +439,12 @@ def cloud_init_interface(name, vm_=None, **kwargs): if ip: fullip = ip if netmask: - fullip += "/{}".format(netmask) + fullip += f"/{netmask}" eth0["ipv4"] = fullip if mac is not None: eth0["mac"] = mac for ix, iopts in enumerate(_cloud_get("additional_ips", [])): - ifh = "eth{}".format(ix + 1) + ifh = f"eth{ix + 1}" ethx = nic_opts.setdefault(ifh, {}) if gw is None: gw = iopts.get("gateway", ethx.get("gateway", None)) @@ -465,7 +465,7 @@ def cloud_init_interface(name, vm_=None, **kwargs): ethx["ipv4"] = aip nm = iopts.get("netmask", "") if nm: - ethx["ipv4"] += "/{}".format(nm) + ethx["ipv4"] += f"/{nm}" for i in ("mac", "hwaddr"): if i in iopts: ethx["mac"] = iopts[i] @@ -543,7 +543,7 @@ def _get_profile(key, name, **kwargs): profile_match = {} else: profile_match = __salt__["config.get"]( - "lxc.{1}:{0}".format(name, key), default=None, merge="recurse" + f"lxc.{key}:{name}", default=None, merge="recurse" ) if profile_match is None: # No matching profile, make the profile an empty dict so that @@ -551,7 +551,7 @@ def _get_profile(key, name, **kwargs): profile_match = {} if not isinstance(profile_match, dict): - raise CommandExecutionError("lxc.{} must be a dictionary".format(key)) + raise CommandExecutionError(f"lxc.{key} must be a dictionary") # Overlay the kwargs to override matched profile data overrides = salt.utils.args.clean_kwargs(**copy.deepcopy(kwargs)) @@ -669,7 +669,7 @@ def _rand_cpu_str(cpu): cpu = int(cpu) avail = __salt__["status.nproc"]() if cpu < avail: - return "0-{}".format(avail) + return f"0-{avail}" to_set = set() while len(to_set) < cpu: choice = random.randint(0, avail - 1) @@ -832,7 +832,7 @@ def _network_conf(conf_tuples=None, **kwargs): "ipv6", ]: continue - ret.append({"lxc.network.{}".format(key): val}) + ret.append({f"lxc.network.{key}": val}) # gateway (in automode) must be appended following network conf ! if not gateway: gateway = args.get("gateway", None) @@ -892,7 +892,7 @@ def _get_lxc_default_data(**kwargs): for k in ["utsname", "rootfs"]: val = kwargs.get(k, None) if val is not None: - ret["lxc.{}".format(k)] = val + ret[f"lxc.{k}"] = val autostart = kwargs.get("autostart") # autostart can have made in kwargs, but with the None # value which is invalid, we need an explicit boolean @@ -1115,7 +1115,7 @@ def _get_base(**kwargs): hash_ = salt.utils.hashutils.get_hash( img_tar, __salt__["config.get"]("hash_type") ) - name = "__base_{}_{}_{}".format(proto, img_name, hash_) + name = f"__base_{proto}_{img_name}_{hash_}" if not exists(name, path=path): create( name, template=template, image=image, path=path, vgname=vgname, **kwargs @@ -1125,11 +1125,11 @@ def _get_base(**kwargs): edit_conf( info(name, path=path)["config"], out_format="commented", - **{"lxc.rootfs": rootfs} + **{"lxc.rootfs": rootfs}, ) return name elif template: - name = "__base_{}".format(template) + name = f"__base_{template}" if not exists(name, path=path): create( name, template=template, image=image, path=path, vgname=vgname, **kwargs @@ -1139,7 +1139,7 @@ def _get_base(**kwargs): edit_conf( info(name, path=path)["config"], out_format="commented", - **{"lxc.rootfs": rootfs} + **{"lxc.rootfs": rootfs}, ) return name return "" @@ -1171,7 +1171,7 @@ def init( bootstrap_args=None, bootstrap_shell=None, bootstrap_url=None, - **kwargs + **kwargs, ): """ Initialize a new container. @@ -1499,7 +1499,7 @@ def init( try: stop(name, path=path) except (SaltInvocationError, CommandExecutionError) as exc: - ret["comment"] = "Unable to stop container: {}".format(exc) + ret["comment"] = f"Unable to stop container: {exc}" if changes: ret["changes"] = changes_dict return ret @@ -1507,7 +1507,7 @@ def init( try: start(name, path=path) except (SaltInvocationError, CommandExecutionError) as exc: - ret["comment"] = "Unable to stop container: {}".format(exc) + ret["comment"] = f"Unable to stop container: {exc}" if changes: ret["changes"] = changes_dict return ret @@ -1515,7 +1515,7 @@ def init( if remove_seed_marker: run( name, - "rm -f '{}'".format(SEED_MARKER), + f"rm -f '{SEED_MARKER}'", path=path, chroot_fallback=False, python_shell=False, @@ -1524,11 +1524,11 @@ def init( # set the default user/password, only the first time if ret.get("result", True) and password: gid = "/.lxc.initial_pass" - gids = [gid, "/lxc.initial_pass", "/.lxc.{}.initial_pass".format(name)] + gids = [gid, "/lxc.initial_pass", f"/.lxc.{name}.initial_pass"] if not any( retcode( name, - 'test -e "{}"'.format(x), + f'test -e "{x}"', chroot_fallback=True, path=path, ignore_retcode=True, @@ -1544,7 +1544,7 @@ def init( default_user not in users and retcode( name, - "id {}".format(default_user), + f"id {default_user}", python_shell=False, path=path, chroot_fallback=True, @@ -1563,7 +1563,7 @@ def init( encrypted=password_encrypted, ) except (SaltInvocationError, CommandExecutionError) as exc: - msg = "{}: Failed to set password".format(user) + exc.strerror + msg = f"{user}: Failed to set password" + exc.strerror # only hardfail in unrecoverable situation: # root cannot be setted up if user == "root": @@ -1591,11 +1591,11 @@ def init( if ret.get("result", True) and dnsservers: # retro compatibility, test also old markers gid = "/.lxc.initial_dns" - gids = [gid, "/lxc.initial_dns", "/lxc.{}.initial_dns".format(name)] + gids = [gid, "/lxc.initial_dns", f"/lxc.{name}.initial_dns"] if not any( retcode( name, - 'test -e "{}"'.format(x), + f'test -e "{x}"', chroot_fallback=True, path=path, ignore_retcode=True, @@ -1628,13 +1628,13 @@ def init( # retro compatibility, test also old markers if remove_seed_marker: - run(name, "rm -f '{}'".format(SEED_MARKER), path=path, python_shell=False) + run(name, f"rm -f '{SEED_MARKER}'", path=path, python_shell=False) gid = "/.lxc.initial_seed" gids = [gid, "/lxc.initial_seed"] if any( retcode( name, - "test -e {}".format(x), + f"test -e {x}", path=path, chroot_fallback=True, ignore_retcode=True, @@ -1703,7 +1703,7 @@ def init( try: stop(name, path=path) except (SaltInvocationError, CommandExecutionError) as exc: - ret["comment"] = "Unable to stop container: {}".format(exc) + ret["comment"] = f"Unable to stop container: {exc}" ret["result"] = False state_post = state(name, path=path) @@ -1711,7 +1711,7 @@ def init( changes.append({"state": {"old": state_pre, "new": state_post}}) if ret.get("result", True): - ret["comment"] = "Container '{}' successfully initialized".format(name) + ret["comment"] = f"Container '{name}' successfully initialized" ret["result"] = True if changes: ret["changes"] = changes_dict @@ -1834,8 +1834,8 @@ def _after_ignition_network_profile(cmd, ret, name, network_profile, path, nic_o # destroy the container if it was partially created cmd = "lxc-destroy" if path: - cmd += " -P {}".format(pipes.quote(path)) - cmd += " -n {}".format(name) + cmd += f" -P {shlex.quote(path)}" + cmd += f" -n {name}" __salt__["cmd.retcode"](cmd, python_shell=False) raise CommandExecutionError( "Container could not be created with cmd '{}': {}".format( @@ -1943,7 +1943,7 @@ def create( # Required params for 'download' template download_template_deps = ("dist", "release", "arch") - cmd = "lxc-create -n {}".format(name) + cmd = f"lxc-create -n {name}" profile = get_container_profile(copy.deepcopy(profile)) kw_overrides = copy.deepcopy(kwargs) @@ -1959,7 +1959,7 @@ def create( path = select("path") if exists(name, path=path): - raise CommandExecutionError("Container '{}' already exists".format(name)) + raise CommandExecutionError(f"Container '{name}' already exists") tvg = select("vgname") vgname = tvg if tvg else __salt__["config.get"]("lxc.vgname") @@ -1997,31 +1997,31 @@ def create( ) options["imgtar"] = img_tar if path: - cmd += " -P {}".format(pipes.quote(path)) + cmd += f" -P {shlex.quote(path)}" if not os.path.exists(path): os.makedirs(path) if config: - cmd += " -f {}".format(config) + cmd += f" -f {config}" if template: - cmd += " -t {}".format(template) + cmd += f" -t {template}" if backing: backing = backing.lower() - cmd += " -B {}".format(backing) + cmd += f" -B {backing}" if backing in ("zfs",): if zfsroot: - cmd += " --zfsroot {}".format(zfsroot) + cmd += f" --zfsroot {zfsroot}" if backing in ("lvm",): if lvname: - cmd += " --lvname {}".format(lvname) + cmd += f" --lvname {lvname}" if vgname: - cmd += " --vgname {}".format(vgname) + cmd += f" --vgname {vgname}" if thinpool: - cmd += " --thinpool {}".format(thinpool) + cmd += f" --thinpool {thinpool}" if backing not in ("dir", "overlayfs"): if fstype: - cmd += " --fstype {}".format(fstype) + cmd += f" --fstype {fstype}" if size: - cmd += " --fssize {}".format(size) + cmd += f" --fssize {size}" if options: if template == "download": @@ -2034,7 +2034,7 @@ def create( ) cmd += " --" for key, val in options.items(): - cmd += " --{} {}".format(key, val) + cmd += f" --{key} {val}" ret = __salt__["cmd.run_all"](cmd, python_shell=False) # please do not merge extra conflicting stuff @@ -2108,13 +2108,11 @@ def clone(name, orig, profile=None, network_profile=None, nic_opts=None, **kwarg path = select("path") if exists(name, path=path): - raise CommandExecutionError("Container '{}' already exists".format(name)) + raise CommandExecutionError(f"Container '{name}' already exists") _ensure_exists(orig, path=path) if state(orig, path=path) != "stopped": - raise CommandExecutionError( - "Container '{}' must be stopped to be cloned".format(orig) - ) + raise CommandExecutionError(f"Container '{orig}' must be stopped to be cloned") backing = select("backing") snapshot = select("snapshot") @@ -2132,21 +2130,21 @@ def clone(name, orig, profile=None, network_profile=None, nic_opts=None, **kwarg if Version(version()) >= Version("2.0"): # https://linuxcontainers.org/lxc/manpages//man1/lxc-copy.1.html cmd = "lxc-copy" - cmd += " {} -n {} -N {}".format(snapshot, orig, name) + cmd += f" {snapshot} -n {orig} -N {name}" else: # https://linuxcontainers.org/lxc/manpages//man1/lxc-clone.1.html cmd = "lxc-clone" - cmd += " {} -o {} -n {}".format(snapshot, orig, name) + cmd += f" {snapshot} -o {orig} -n {name}" if path: - cmd += " -P {}".format(pipes.quote(path)) + cmd += f" -P {shlex.quote(path)}" if not os.path.exists(path): os.makedirs(path) if backing: backing = backing.lower() - cmd += " -B {}".format(backing) + cmd += f" -B {backing}" if backing not in ("dir", "overlayfs"): if size: - cmd += " -L {}".format(size) + cmd += f" -L {size}" ret = __salt__["cmd.run_all"](cmd, python_shell=False) # please do not merge extra conflicting stuff # inside those two line (ret =, return) @@ -2177,7 +2175,7 @@ def ls_(active=None, cache=True, path=None): salt '*' lxc.ls salt '*' lxc.ls active=True """ - contextvar = "lxc.ls{}".format(path) + contextvar = f"lxc.ls{path}" if active: contextvar += ".active" if cache and (contextvar in __context__): @@ -2186,7 +2184,7 @@ def ls_(active=None, cache=True, path=None): ret = [] cmd = "lxc-ls" if path: - cmd += " -P {}".format(pipes.quote(path)) + cmd += f" -P {shlex.quote(path)}" if active: cmd += " --active" output = __salt__["cmd.run_stdout"](cmd, python_shell=False) @@ -2242,8 +2240,8 @@ def list_(extra=False, limit=None, path=None): for container in ctnrs: cmd = "lxc-info" if path: - cmd += " -P {}".format(pipes.quote(path)) - cmd += " -n {}".format(container) + cmd += f" -P {shlex.quote(path)}" + cmd += f" -n {container}" c_info = __salt__["cmd.run"](cmd, python_shell=False, output_loglevel="debug") c_state = None for line in c_info.splitlines(): @@ -2294,20 +2292,20 @@ def _change_state( return { "result": True, "state": {"old": expected, "new": expected}, - "comment": "Container '{}' already {}".format(name, expected), + "comment": f"Container '{name}' already {expected}", } if cmd == "lxc-destroy": # Kill the container first scmd = "lxc-stop" if path: - scmd += " -P {}".format(pipes.quote(path)) - scmd += " -k -n {}".format(name) + scmd += f" -P {shlex.quote(path)}" + scmd += f" -k -n {name}" __salt__["cmd.run"](scmd, python_shell=False) if path and " -P " not in cmd: - cmd += " -P {}".format(pipes.quote(path)) - cmd += " -n {}".format(name) + cmd += f" -P {shlex.quote(path)}" + cmd += f" -n {name}" # certain lxc commands need to be taken with care (lxc-start) # as te command itself mess with double forks; we must not @@ -2337,8 +2335,8 @@ def _change_state( # some commands do not wait, so we will rcmd = "lxc-wait" if path: - rcmd += " -P {}".format(pipes.quote(path)) - rcmd += " -n {} -s {}".format(name, expected.upper()) + rcmd += f" -P {shlex.quote(path)}" + rcmd += f" -n {name} -s {expected.upper()}" __salt__["cmd.run"](rcmd, python_shell=False, timeout=30) _clear_context() post = state(name, path=path) @@ -2351,7 +2349,7 @@ def _ensure_exists(name, path=None): Raise an exception if the container does not exist """ if not exists(name, path=path): - raise CommandExecutionError("Container '{}' does not exist".format(name)) + raise CommandExecutionError(f"Container '{name}' does not exist") def _ensure_running(name, no_start=False, path=None): @@ -2373,11 +2371,11 @@ def _ensure_running(name, no_start=False, path=None): return start(name, path=path) elif pre == "stopped": if no_start: - raise CommandExecutionError("Container '{}' is not running".format(name)) + raise CommandExecutionError(f"Container '{name}' is not running") return start(name, path=path) elif pre == "frozen": if no_start: - raise CommandExecutionError("Container '{}' is not running".format(name)) + raise CommandExecutionError(f"Container '{name}' is not running") return unfreeze(name, path=path) @@ -2459,13 +2457,11 @@ def start(name, **kwargs): lxc_config = os.path.join(cpath, name, "config") # we try to start, even without config, if global opts are there if os.path.exists(lxc_config): - cmd += " -f {}".format(pipes.quote(lxc_config)) + cmd += f" -f {shlex.quote(lxc_config)}" cmd += " -d" _ensure_exists(name, path=path) if state(name, path=path) == "frozen": - raise CommandExecutionError( - "Container '{}' is frozen, use lxc.unfreeze".format(name) - ) + raise CommandExecutionError(f"Container '{name}' is frozen, use lxc.unfreeze") # lxc-start daemonize itself violently, we must not communicate with it use_vt = kwargs.get("use_vt", None) with_communicate = kwargs.get("with_communicate", False) @@ -2560,11 +2556,11 @@ def freeze(name, **kwargs): start_ = kwargs.get("start", False) if orig_state == "stopped": if not start_: - raise CommandExecutionError("Container '{}' is stopped".format(name)) + raise CommandExecutionError(f"Container '{name}' is stopped") start(name, path=path) cmd = "lxc-freeze" if path: - cmd += " -P {}".format(pipes.quote(path)) + cmd += f" -P {shlex.quote(path)}" ret = _change_state(cmd, name, "frozen", use_vt=use_vt, path=path) if orig_state == "stopped" and start_: ret["state"]["old"] = orig_state @@ -2596,10 +2592,10 @@ def unfreeze(name, path=None, use_vt=None): """ _ensure_exists(name, path=path) if state(name, path=path) == "stopped": - raise CommandExecutionError("Container '{}' is stopped".format(name)) + raise CommandExecutionError(f"Container '{name}' is stopped") cmd = "lxc-unfreeze" if path: - cmd += " -P {}".format(pipes.quote(path)) + cmd += f" -P {shlex.quote(path)}" return _change_state(cmd, name, "running", path=path, use_vt=use_vt) @@ -2635,7 +2631,7 @@ def destroy(name, stop=False, path=None): """ _ensure_exists(name, path=path) if not stop and state(name, path=path) != "stopped": - raise CommandExecutionError("Container '{}' is not stopped".format(name)) + raise CommandExecutionError(f"Container '{name}' is not stopped") return _change_state("lxc-destroy", name, None, path=path) @@ -2684,7 +2680,7 @@ def state(name, path=None): """ # Don't use _ensure_exists() here, it will mess with _change_state() - cachekey = "lxc.state.{}{}".format(name, path) + cachekey = f"lxc.state.{name}{path}" try: return __context__[cachekey] except KeyError: @@ -2693,13 +2689,13 @@ def state(name, path=None): else: cmd = "lxc-info" if path: - cmd += " -P {}".format(pipes.quote(path)) - cmd += " -n {}".format(name) + cmd += f" -P {shlex.quote(path)}" + cmd += f" -n {name}" ret = __salt__["cmd.run_all"](cmd, python_shell=False) if ret["retcode"] != 0: _clear_context() raise CommandExecutionError( - "Unable to get state of container '{}'".format(name) + f"Unable to get state of container '{name}'" ) c_infos = ret["stdout"].splitlines() c_state = None @@ -2731,13 +2727,11 @@ def get_parameter(name, parameter, path=None): _ensure_exists(name, path=path) cmd = "lxc-cgroup" if path: - cmd += " -P {}".format(pipes.quote(path)) - cmd += " -n {} {}".format(name, parameter) + cmd += f" -P {shlex.quote(path)}" + cmd += f" -n {name} {parameter}" ret = __salt__["cmd.run_all"](cmd, python_shell=False) if ret["retcode"] != 0: - raise CommandExecutionError( - "Unable to retrieve value for '{}'".format(parameter) - ) + raise CommandExecutionError(f"Unable to retrieve value for '{parameter}'") return ret["stdout"].strip() @@ -2762,8 +2756,8 @@ def set_parameter(name, parameter, value, path=None): cmd = "lxc-cgroup" if path: - cmd += " -P {}".format(pipes.quote(path)) - cmd += " -n {} {} {}".format(name, parameter, value) + cmd += f" -P {shlex.quote(path)}" + cmd += f" -n {name} {parameter} {value}" ret = __salt__["cmd.run_all"](cmd, python_shell=False) if ret["retcode"] != 0: return False @@ -2787,7 +2781,7 @@ def info(name, path=None): salt '*' lxc.info name """ - cachekey = "lxc.info.{}{}".format(name, path) + cachekey = f"lxc.info.{name}{path}" try: return __context__[cachekey] except KeyError: @@ -2799,9 +2793,7 @@ def info(name, path=None): conf_file = os.path.join(cpath, str(name), "config") if not os.path.isfile(conf_file): - raise CommandExecutionError( - "LXC config file {} does not exist".format(conf_file) - ) + raise CommandExecutionError(f"LXC config file {conf_file} does not exist") ret = {} config = [] @@ -3000,9 +2992,7 @@ def update_lxc_conf(name, lxc_conf, lxc_conf_unset, path=None): cpath = get_root_path(path) lxc_conf_p = os.path.join(cpath, name, "config") if not os.path.exists(lxc_conf_p): - raise SaltInvocationError( - "Configuration file {} does not exist".format(lxc_conf_p) - ) + raise SaltInvocationError(f"Configuration file {lxc_conf_p} does not exist") changes = {"edited": [], "added": [], "removed": []} ret = {"changes": changes, "result": True, "comment": ""} @@ -3054,17 +3044,15 @@ def update_lxc_conf(name, lxc_conf, lxc_conf_unset, path=None): conf = "" for key, val in dest_lxc_conf: if not val: - conf += "{}\n".format(key) + conf += f"{key}\n" else: - conf += "{} = {}\n".format(key.strip(), val.strip()) + conf += f"{key.strip()} = {val.strip()}\n" conf_changed = conf != orig_config chrono = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") if conf_changed: # DO NOT USE salt.utils.files.fopen here, i got (kiorky) # problems with lxc configs which were wiped ! - with salt.utils.files.fopen( - "{}.{}".format(lxc_conf_p, chrono), "w" - ) as wfic: + with salt.utils.files.fopen(f"{lxc_conf_p}.{chrono}", "w") as wfic: wfic.write(salt.utils.stringutils.to_str(conf)) with salt.utils.files.fopen(lxc_conf_p, "w") as wfic: wfic.write(salt.utils.stringutils.to_str(conf)) @@ -3113,8 +3101,8 @@ def set_dns(name, dnsservers=None, searchdomains=None, path=None): searchdomains = searchdomains.split(",") except AttributeError: raise SaltInvocationError("Invalid input for 'searchdomains' parameter") - dns = ["nameserver {}".format(x) for x in dnsservers] - dns.extend(["search {}".format(x) for x in searchdomains]) + dns = [f"nameserver {x}" for x in dnsservers] + dns.extend([f"search {x}" for x in searchdomains]) dns = "\n".join(dns) + "\n" # we may be using resolvconf in the container # We need to handle that case with care: @@ -3129,7 +3117,7 @@ def set_dns(name, dnsservers=None, searchdomains=None, path=None): # - We finally also set /etc/resolv.conf in all cases rstr = __salt__["test.random_hash"]() # no tmp here, apparmor won't let us execute ! - script = "/sbin/{}_dns.sh".format(rstr) + script = f"/sbin/{rstr}_dns.sh" DNS_SCRIPT = "\n".join( [ # 'set -x', @@ -3153,7 +3141,7 @@ def set_dns(name, dnsservers=None, searchdomains=None, path=None): ] ) result = run_all( - name, "tee {}".format(script), path=path, stdin=DNS_SCRIPT, python_shell=True + name, f"tee {script}", path=path, stdin=DNS_SCRIPT, python_shell=True ) if result["retcode"] == 0: result = run_all( @@ -3170,7 +3158,7 @@ def set_dns(name, dnsservers=None, searchdomains=None, path=None): python_shell=True, ) if result["retcode"] != 0: - error = "Unable to write to /etc/resolv.conf in container '{}'".format(name) + error = f"Unable to write to /etc/resolv.conf in container '{name}'" if result["stderr"]: error += ": {}".format(result["stderr"]) raise CommandExecutionError(error) @@ -3193,12 +3181,12 @@ def running_systemd(name, cache=True, path=None): salt '*' lxc.running_systemd ubuntu """ - k = "lxc.systemd.test.{}{}".format(name, path) + k = f"lxc.systemd.test.{name}{path}" ret = __context__.get(k, None) if ret is None or not cache: rstr = __salt__["test.random_hash"]() # no tmp here, apparmor won't let us execute ! - script = "/sbin/{}_testsystemd.sh".format(rstr) + script = f"/sbin/{rstr}_testsystemd.sh" # ubuntu already had since trusty some bits of systemd but was # still using upstart ... # we need to be a bit more careful that just testing that systemd @@ -3227,7 +3215,7 @@ def running_systemd(name, cache=True, path=None): """ ) result = run_all( - name, "tee {}".format(script), path=path, stdin=_script, python_shell=True + name, f"tee {script}", path=path, stdin=_script, python_shell=True ) if result["retcode"] == 0: result = run_all( @@ -3237,9 +3225,7 @@ def running_systemd(name, cache=True, path=None): python_shell=True, ) else: - raise CommandExecutionError( - "lxc {} failed to copy initd tester".format(name) - ) + raise CommandExecutionError(f"lxc {name} failed to copy initd tester") run_all( name, 'sh -c \'if [ -f "{0}" ];then rm -f "{0}";fi\''.format(script), @@ -3361,9 +3347,9 @@ def wait_started(name, path=None, timeout=300): """ if not exists(name, path=path): - raise CommandExecutionError("Container {} does does exists".format(name)) + raise CommandExecutionError(f"Container {name} does does exists") if not state(name, path=path) == "running": - raise CommandExecutionError("Container {} is not running".format(name)) + raise CommandExecutionError(f"Container {name} is not running") ret = False if running_systemd(name, path=path): test_started = test_sd_started_state @@ -3520,7 +3506,7 @@ def bootstrap( seeded = ( retcode( name, - "test -e '{}'".format(SEED_MARKER), + f"test -e '{SEED_MARKER}'", path=path, chroot_fallback=True, ignore_retcode=True, @@ -3543,9 +3529,9 @@ def bootstrap( if needs_install or force_install or unconditional_install: if install: rstr = __salt__["test.random_hash"]() - configdir = "/var/tmp/.c_{}".format(rstr) + configdir = f"/var/tmp/.c_{rstr}" - cmd = "install -m 0700 -d {}".format(configdir) + cmd = f"install -m 0700 -d {configdir}" if run_all(name, cmd, path=path, python_shell=False)["retcode"] != 0: log.error("tmpdir %s creation failed %s", configdir, cmd) return False @@ -3553,11 +3539,11 @@ def bootstrap( bs_ = __salt__["config.gather_bootstrap_script"]( bootstrap=bootstrap_url ) - script = "/sbin/{}_bootstrap.sh".format(rstr) + script = f"/sbin/{rstr}_bootstrap.sh" copy_to(name, bs_, script, path=path) result = run_all( name, - 'sh -c "chmod +x {}"'.format(script), + f'sh -c "chmod +x {script}"', path=path, python_shell=True, ) @@ -3631,7 +3617,7 @@ def bootstrap( freeze(name, path=path) # mark seeded upon successful install if ret: - run(name, "touch '{}'".format(SEED_MARKER), path=path, python_shell=False) + run(name, f"touch '{SEED_MARKER}'", path=path, python_shell=False) return ret @@ -3652,7 +3638,7 @@ def attachable(name, path=None): salt 'minion' lxc.attachable ubuntu """ - cachekey = "lxc.attachable{}{}".format(name, path) + cachekey = f"lxc.attachable{name}{path}" try: return __context__[cachekey] except KeyError: @@ -3662,8 +3648,8 @@ def attachable(name, path=None): log.debug("Checking if LXC container %s is attachable", name) cmd = "lxc-attach" if path: - cmd += " -P {}".format(pipes.quote(path)) - cmd += " --clear-env -n {} -- /usr/bin/env".format(name) + cmd += f" -P {shlex.quote(path)}" + cmd += f" --clear-env -n {name} -- /usr/bin/env" result = ( __salt__["cmd.retcode"]( cmd, python_shell=False, output_loglevel="quiet", ignore_retcode=True @@ -3719,7 +3705,7 @@ def _run( ) else: if not chroot_fallback: - raise CommandExecutionError("{} is not attachable.".format(name)) + raise CommandExecutionError(f"{name} is not attachable.") rootfs = info(name, path=path).get("rootfs") # Set context var to make cmd.run_chroot run cmd.run instead of # cmd.run_all. @@ -4214,7 +4200,7 @@ def _get_md5(name, path): Get the MD5 checksum of a file from a container """ output = run_stdout( - name, 'md5sum "{}"'.format(path), chroot_fallback=True, ignore_retcode=True + name, f'md5sum "{path}"', chroot_fallback=True, ignore_retcode=True ) try: return output.split()[0] @@ -4381,7 +4367,7 @@ def write_conf(conf_file, conf): line[key], (str, (str,), (int,), float), ): - out_line = " = ".join((key, "{}".format(line[key]))) + out_line = " = ".join((key, f"{line[key]}")) elif isinstance(line[key], dict): out_line = " = ".join((key, line[key]["value"])) if "comment" in line[key]: @@ -4474,7 +4460,7 @@ def edit_conf( net_changes = _config_list( conf, only_net=True, - **{"network_profile": DEFAULT_NIC, "nic_opts": nic_opts} + **{"network_profile": DEFAULT_NIC, "nic_opts": nic_opts}, ) if net_changes: lxc_config.extend(net_changes) @@ -4524,20 +4510,20 @@ def reboot(name, path=None): salt 'minion' lxc.reboot myvm """ - ret = {"result": True, "changes": {}, "comment": "{} rebooted".format(name)} + ret = {"result": True, "changes": {}, "comment": f"{name} rebooted"} does_exist = exists(name, path=path) if does_exist and (state(name, path=path) == "running"): try: stop(name, path=path) except (SaltInvocationError, CommandExecutionError) as exc: - ret["comment"] = "Unable to stop container: {}".format(exc) + ret["comment"] = f"Unable to stop container: {exc}" ret["result"] = False return ret if does_exist and (state(name, path=path) != "running"): try: start(name, path=path) except (SaltInvocationError, CommandExecutionError) as exc: - ret["comment"] = "Unable to stop container: {}".format(exc) + ret["comment"] = f"Unable to stop container: {exc}" ret["result"] = False return ret ret["changes"][name] = "rebooted" @@ -4559,7 +4545,7 @@ def reconfigure( utsname=None, rootfs=None, path=None, - **kwargs + **kwargs, ): """ Reconfigure a container. @@ -4625,7 +4611,7 @@ def reconfigure( path = os.path.join(cpath, name, "config") ret = { "name": name, - "comment": "config for {} up to date".format(name), + "comment": f"config for {name} up to date", "result": True, "changes": changes, } @@ -4677,7 +4663,7 @@ def reconfigure( edit_conf(path, out_format="commented", lxc_config=new_cfg) chunks = read_conf(path, out_format="commented") if old_chunks != chunks: - ret["comment"] = "{} lxc config updated".format(name) + ret["comment"] = f"{name} lxc config updated" if state(name, path=path) == "running": cret = reboot(name, path=path) ret["result"] = cret["result"] @@ -4763,9 +4749,9 @@ def get_pid(name, path=None): """ if name not in list_(limit="running", path=path): raise CommandExecutionError( - "Container {} is not running, can't determine PID".format(name) + f"Container {name} is not running, can't determine PID" ) - info = __salt__["cmd.run"]("lxc-info -n {}".format(name)).split("\n") + info = __salt__["cmd.run"](f"lxc-info -n {name}").split("\n") pid = [ line.split(":")[1].strip() for line in info @@ -4812,21 +4798,19 @@ def add_veth(name, interface_name, bridge=None, path=None): raise CommandExecutionError( "Directory /var/run required for lxc.add_veth doesn't exists" ) - if not __salt__["file.file_exists"]("/proc/{}/ns/net".format(pid)): + if not __salt__["file.file_exists"](f"/proc/{pid}/ns/net"): raise CommandExecutionError( - "Proc file for container {} network namespace doesn't exists".format(name) + f"Proc file for container {name} network namespace doesn't exists" ) if not __salt__["file.directory_exists"]("/var/run/netns"): __salt__["file.mkdir"]("/var/run/netns") # Ensure that the symlink is up to date (change on container restart) - if __salt__["file.is_link"]("/var/run/netns/{}".format(name)): - __salt__["file.remove"]("/var/run/netns/{}".format(name)) + if __salt__["file.is_link"](f"/var/run/netns/{name}"): + __salt__["file.remove"](f"/var/run/netns/{name}") - __salt__["file.symlink"]( - "/proc/{}/ns/net".format(pid), "/var/run/netns/{}".format(name) - ) + __salt__["file.symlink"](f"/proc/{pid}/ns/net", f"/var/run/netns/{name}") # Ensure that interface doesn't exists interface_exists = 0 == __salt__["cmd.retcode"]( @@ -4851,12 +4835,10 @@ def add_veth(name, interface_name, bridge=None, path=None): ) != 0 ): + raise CommandExecutionError(f"Error while creating the veth pair {random_veth}") + if __salt__["cmd.retcode"](f"ip link set dev {random_veth} up") != 0: raise CommandExecutionError( - "Error while creating the veth pair {}".format(random_veth) - ) - if __salt__["cmd.retcode"]("ip link set dev {} up".format(random_veth)) != 0: - raise CommandExecutionError( - "Error while bringing up host-side veth {}".format(random_veth) + f"Error while bringing up host-side veth {random_veth}" ) # Attach it to the container @@ -4872,7 +4854,7 @@ def add_veth(name, interface_name, bridge=None, path=None): ) ) - __salt__["file.remove"]("/var/run/netns/{}".format(name)) + __salt__["file.remove"](f"/var/run/netns/{name}") if bridge is not None: __salt__["bridge.addif"](bridge, random_veth) diff --git a/salt/modules/mac_keychain.py b/salt/modules/mac_keychain.py index a823c428b76..7fdc162b9aa 100644 --- a/salt/modules/mac_keychain.py +++ b/salt/modules/mac_keychain.py @@ -11,20 +11,6 @@ import shlex import salt.utils.platform -try: - import pipes - - HAS_DEPS = True -except ImportError: - HAS_DEPS = False - -if hasattr(shlex, "quote"): - _quote = shlex.quote -elif HAS_DEPS and hasattr(pipes, "quote"): - _quote = pipes.quote -else: - _quote = None - log = logging.getLogger(__name__) __virtualname__ = "keychain" @@ -34,7 +20,7 @@ def __virtual__(): """ Only work on Mac OS """ - if salt.utils.platform.is_darwin() and _quote is not None: + if salt.utils.platform.is_darwin(): return __virtualname__ return (False, "Only available on Mac OS systems with pipes") @@ -82,7 +68,7 @@ def install( if keychain_password is not None: unlock_keychain(keychain, keychain_password) - cmd = "security import {} -P {} -k {}".format(cert, password, keychain) + cmd = f"security import {cert} -P {password} -k {keychain}" if allow_any: cmd += " -A" return __salt__["cmd.run"](cmd) @@ -117,7 +103,7 @@ def uninstall( if keychain_password is not None: unlock_keychain(keychain, keychain_password) - cmd = 'security delete-certificate -c "{}" {}'.format(cert_name, keychain) + cmd = f'security delete-certificate -c "{cert_name}" {keychain}' return __salt__["cmd.run"](cmd) @@ -137,7 +123,7 @@ def list_certs(keychain="/Library/Keychains/System.keychain"): """ cmd = ( 'security find-certificate -a {} | grep -o "alis".*\\" | ' - "grep -o '\\\"[-A-Za-z0-9.:() ]*\\\"'".format(_quote(keychain)) + "grep -o '\\\"[-A-Za-z0-9.:() ]*\\\"'".format(shlex.quote(keychain)) ) out = __salt__["cmd.run"](cmd, python_shell=True) return out.replace('"', "").split("\n") @@ -165,7 +151,7 @@ def get_friendly_name(cert, password): """ cmd = ( "openssl pkcs12 -in {} -passin pass:{} -info -nodes -nokeys 2> /dev/null | " - "grep friendlyName:".format(_quote(cert), _quote(password)) + "grep friendlyName:".format(shlex.quote(cert), shlex.quote(password)) ) out = __salt__["cmd.run"](cmd, python_shell=True) return out.replace("friendlyName: ", "").strip() @@ -187,7 +173,7 @@ def get_default_keychain(user=None, domain="user"): salt '*' keychain.get_default_keychain """ - cmd = "security default-keychain -d {}".format(domain) + cmd = f"security default-keychain -d {domain}" return __salt__["cmd.run"](cmd, runas=user) @@ -210,7 +196,7 @@ def set_default_keychain(keychain, domain="user", user=None): salt '*' keychain.set_keychain /Users/fred/Library/Keychains/login.keychain """ - cmd = "security default-keychain -d {} -s {}".format(domain, keychain) + cmd = f"security default-keychain -d {domain} -s {keychain}" return __salt__["cmd.run"](cmd, runas=user) @@ -233,7 +219,7 @@ def unlock_keychain(keychain, password): salt '*' keychain.unlock_keychain /tmp/test.p12 test123 """ - cmd = "security unlock-keychain -p {} {}".format(password, keychain) + cmd = f"security unlock-keychain -p {password} {keychain}" __salt__["cmd.run"](cmd) @@ -261,7 +247,7 @@ def get_hash(name, password=None): name, password ) else: - cmd = 'security find-certificate -c "{}" -m -p'.format(name) + cmd = f'security find-certificate -c "{name}" -m -p' out = __salt__["cmd.run"](cmd) matches = re.search( diff --git a/salt/modules/macpackage.py b/salt/modules/macpackage.py index faf5810d4fc..f9a6b7bb95c 100644 --- a/salt/modules/macpackage.py +++ b/salt/modules/macpackage.py @@ -9,31 +9,16 @@ import shlex import salt.utils.platform -try: - import pipes - - HAS_DEPS = True -except ImportError: - HAS_DEPS = False - - log = logging.getLogger(__name__) + __virtualname__ = "macpackage" -if hasattr(shlex, "quote"): - _quote = shlex.quote -elif HAS_DEPS and hasattr(pipes, "quote"): - _quote = pipes.quote -else: - _quote = None - - def __virtual__(): """ Only work on Mac OS """ - if salt.utils.platform.is_darwin() and _quote is not None: + if salt.utils.platform.is_darwin(): return __virtualname__ return (False, "Only available on Mac OS systems with pipes") @@ -60,11 +45,11 @@ def install(pkg, target="LocalSystem", store=False, allow_untrusted=False): """ if "*." not in pkg: # If we use wildcards, we cannot use quotes - pkg = _quote(pkg) + pkg = shlex.quote(pkg) - target = _quote(target) + target = shlex.quote(target) - cmd = "installer -pkg {} -target {}".format(pkg, target) + cmd = f"installer -pkg {pkg} -target {target}" if store: cmd += " -store" if allow_untrusted: @@ -109,7 +94,7 @@ def install_app(app, target="/Applications/"): if not app[-1] == "/": app += "/" - cmd = 'rsync -a --delete "{}" "{}"'.format(app, target) + cmd = f'rsync -a --delete "{app}" "{target}"' return __salt__["cmd.run"](cmd) @@ -154,7 +139,7 @@ def mount(dmg): temp_dir = __salt__["temp.dir"](prefix="dmg-") - cmd = 'hdiutil attach -readonly -nobrowse -mountpoint {} "{}"'.format(temp_dir, dmg) + cmd = f'hdiutil attach -readonly -nobrowse -mountpoint {temp_dir} "{dmg}"' return __salt__["cmd.run"](cmd), temp_dir @@ -176,7 +161,7 @@ def unmount(mountpoint): salt '*' macpackage.unmount /dev/disk2 """ - cmd = 'hdiutil detach "{}"'.format(mountpoint) + cmd = f'hdiutil detach "{mountpoint}"' return __salt__["cmd.run"](cmd) @@ -216,7 +201,7 @@ def get_pkg_id(pkg): salt '*' macpackage.get_pkg_id /tmp/test.pkg """ - pkg = _quote(pkg) + pkg = shlex.quote(pkg) package_ids = [] # Create temp directory @@ -224,7 +209,7 @@ def get_pkg_id(pkg): try: # List all of the PackageInfo files - cmd = "xar -t -f {} | grep PackageInfo".format(pkg) + cmd = f"xar -t -f {pkg} | grep PackageInfo" out = __salt__["cmd.run"](cmd, python_shell=True, output_loglevel="quiet") files = out.split("\n") @@ -264,12 +249,12 @@ def get_mpkg_ids(mpkg): salt '*' macpackage.get_mpkg_ids /dev/disk2 """ - mpkg = _quote(mpkg) + mpkg = shlex.quote(mpkg) package_infos = [] base_path = os.path.dirname(mpkg) # List all of the .pkg files - cmd = "find {} -name *.pkg".format(base_path) + cmd = f"find {base_path} -name *.pkg" out = __salt__["cmd.run"](cmd, python_shell=True) pkg_files = out.split("\n") @@ -281,7 +266,7 @@ def get_mpkg_ids(mpkg): def _get_pkg_id_from_pkginfo(pkginfo): # Find our identifiers - pkginfo = _quote(pkginfo) + pkginfo = shlex.quote(pkginfo) cmd = "cat {} | grep -Eo 'identifier=\"[a-zA-Z.0-9\\-]*\"' | cut -c 13- | tr -d '\"'".format( pkginfo ) @@ -294,8 +279,8 @@ def _get_pkg_id_from_pkginfo(pkginfo): def _get_pkg_id_dir(path): - path = _quote(os.path.join(path, "Contents/Info.plist")) - cmd = '/usr/libexec/PlistBuddy -c "print :CFBundleIdentifier" {}'.format(path) + path = shlex.quote(os.path.join(path, "Contents/Info.plist")) + cmd = f'/usr/libexec/PlistBuddy -c "print :CFBundleIdentifier" {path}' # We can only use wildcards in python_shell which is # sent by the macpackage state diff --git a/salt/modules/openstack_config.py b/salt/modules/openstack_config.py index 823afbf1c60..937c10da61a 100644 --- a/salt/modules/openstack_config.py +++ b/salt/modules/openstack_config.py @@ -13,28 +13,11 @@ import shlex import salt.exceptions import salt.utils.decorators.path -try: - import pipes - - HAS_DEPS = True -except ImportError: - HAS_DEPS = False - -if hasattr(shlex, "quote"): - _quote = shlex.quote -elif HAS_DEPS and hasattr(pipes, "quote"): - _quote = pipes.quote -else: - _quote = None - - # Don't shadow built-in's. __func_alias__ = {"set_": "set"} def __virtual__(): - if _quote is None and not HAS_DEPS: - return (False, "Missing dependencies") return True @@ -69,10 +52,10 @@ def set_(filename, section, parameter, value): salt-call openstack_config.set /etc/keystone/keystone.conf sql connection foo """ - filename = _quote(filename) - section = _quote(section) - parameter = _quote(parameter) - value = _quote(str(value)) + filename = shlex.quote(filename) + section = shlex.quote(section) + parameter = shlex.quote(parameter) + value = shlex.quote(str(value)) result = __salt__["cmd.run_all"]( "openstack-config --set {} {} {} {}".format( @@ -109,12 +92,12 @@ def get(filename, section, parameter): """ - filename = _quote(filename) - section = _quote(section) - parameter = _quote(parameter) + filename = shlex.quote(filename) + section = shlex.quote(section) + parameter = shlex.quote(parameter) result = __salt__["cmd.run_all"]( - "openstack-config --get {} {} {}".format(filename, section, parameter), + f"openstack-config --get {filename} {section} {parameter}", python_shell=False, ) @@ -145,12 +128,12 @@ def delete(filename, section, parameter): salt-call openstack_config.delete /etc/keystone/keystone.conf sql connection """ - filename = _quote(filename) - section = _quote(section) - parameter = _quote(parameter) + filename = shlex.quote(filename) + section = shlex.quote(section) + parameter = shlex.quote(parameter) result = __salt__["cmd.run_all"]( - "openstack-config --del {} {} {}".format(filename, section, parameter), + f"openstack-config --del {filename} {section} {parameter}", python_shell=False, ) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index 25a72f1063c..f73959a92ed 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -46,8 +46,8 @@ import hmac import io import logging import os -import pipes import re +import shlex import tempfile import salt.utils.files @@ -136,7 +136,7 @@ def __virtual__(): for util in utils: if not salt.utils.path.which(util): if not _find_pg_binary(util): - return (False, "{} was not found".format(util)) + return (False, f"{util} was not found") return True @@ -241,14 +241,14 @@ def _run_initdb( raise CommandExecutionError("initdb executable not found.") cmd = [ _INITDB_BIN, - "--pgdata={}".format(name), - "--username={}".format(user), - "--auth={}".format(auth), - "--encoding={}".format(encoding), + f"--pgdata={name}", + f"--username={user}", + f"--auth={auth}", + f"--encoding={encoding}", ] if locale is not None: - cmd.append("--locale={}".format(locale)) + cmd.append(f"--locale={locale}") # intentionally use short option, as the long option name has been # renamed from "xlogdir" to "waldir" in PostgreSQL 10 @@ -262,9 +262,9 @@ def _run_initdb( if password is not None: pgpassfile = salt.utils.files.mkstemp(text=True) with salt.utils.files.fopen(pgpassfile, "w") as fp_: - fp_.write(salt.utils.stringutils.to_str("{}".format(password))) + fp_.write(salt.utils.stringutils.to_str(f"{password}")) __salt__["file.chown"](pgpassfile, runas, "") - cmd.extend(["--pwfile={}".format(pgpassfile)]) + cmd.extend([f"--pwfile={pgpassfile}"]) kwargs = dict( runas=runas, @@ -273,7 +273,7 @@ def _run_initdb( "postgres.timeout", default=_DEFAULT_COMMAND_TIMEOUT_SECS ), ) - cmdstr = " ".join([pipes.quote(c) for c in cmd]) + cmdstr = " ".join([shlex.quote(c) for c in cmd]) ret = __salt__["cmd.run_all"](cmdstr, python_shell=False, **kwargs) if ret.get("retcode", 0) != 0: @@ -582,9 +582,7 @@ def _quote_ddl_value(value, quote="'"): if value is None: return None if quote in value: # detect trivial sqli - raise SaltInvocationError( - "Unsupported character {} in value: {}".format(quote, value) - ) + raise SaltInvocationError(f"Unsupported character {quote} in value: {value}") return "{quote}{value}{quote}".format(quote=quote, value=value) @@ -617,7 +615,7 @@ def db_create( """ # Base query to create a database - query = 'CREATE DATABASE "{}"'.format(name) + query = f'CREATE DATABASE "{name}"' # "With"-options to create a database with_args = salt.utils.odict.OrderedDict( @@ -685,11 +683,9 @@ def db_alter( else: queries = [] if owner: - queries.append('ALTER DATABASE "{}" OWNER TO "{}"'.format(name, owner)) + queries.append(f'ALTER DATABASE "{name}" OWNER TO "{owner}"') if tablespace: - queries.append( - 'ALTER DATABASE "{}" SET TABLESPACE "{}"'.format(name, tablespace) - ) + queries.append(f'ALTER DATABASE "{name}" SET TABLESPACE "{tablespace}"') for query in queries: ret = _psql_prepare_and_run( ["-c", query], @@ -726,10 +722,10 @@ def db_remove( salt '*' postgres.db_remove 'dbname' """ for query in [ - 'REVOKE CONNECT ON DATABASE "{db}" FROM public;'.format(db=name), + f'REVOKE CONNECT ON DATABASE "{name}" FROM public;', "SELECT pid, pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname =" " '{db}' AND pid <> pg_backend_pid();".format(db=name), - 'DROP DATABASE "{db}";'.format(db=name), + f'DROP DATABASE "{name}";', ]: ret = _psql_prepare_and_run( ["-c", query], @@ -741,7 +737,7 @@ def db_remove( password=password, ) if ret["retcode"] != 0: - raise Exception("Failed: ret={}".format(ret)) + raise Exception(f"Failed: ret={ret}") return True @@ -846,10 +842,10 @@ def tablespace_create( owner_query = "" options_query = "" if owner: - owner_query = 'OWNER "{}"'.format(owner) + owner_query = f'OWNER "{owner}"' # should come out looking like: 'OWNER postgres' if options: - optionstext = ["{} = {}".format(k, v) for k, v in options.items()] + optionstext = [f"{k} = {v}" for k, v in options.items()] options_query = "WITH ( {} )".format(", ".join(optionstext)) # should come out looking like: 'WITH ( opt1 = 1.0, opt2 = 4.0 )' query = "CREATE TABLESPACE \"{}\" {} LOCATION '{}' {}".format( @@ -902,9 +898,9 @@ def tablespace_alter( queries = [] if new_name: - queries.append('ALTER TABLESPACE "{}" RENAME TO "{}"'.format(name, new_name)) + queries.append(f'ALTER TABLESPACE "{name}" RENAME TO "{new_name}"') if new_owner: - queries.append('ALTER TABLESPACE "{}" OWNER TO "{}"'.format(name, new_owner)) + queries.append(f'ALTER TABLESPACE "{name}" OWNER TO "{new_owner}"') if set_option: queries.append( 'ALTER TABLESPACE "{}" SET ({} = {})'.format( @@ -912,7 +908,7 @@ def tablespace_alter( ) ) if reset_option: - queries.append('ALTER TABLESPACE "{}" RESET ({})'.format(name, reset_option)) + queries.append(f'ALTER TABLESPACE "{name}" RESET ({reset_option})') for query in queries: ret = _psql_prepare_and_run( @@ -950,7 +946,7 @@ def tablespace_remove( .. versionadded:: 2015.8.0 """ - query = 'DROP TABLESPACE "{}"'.format(name) + query = f'DROP TABLESPACE "{name}"' ret = _psql_prepare_and_run( ["-c", query], user=user, @@ -1158,11 +1154,11 @@ def _add_role_flag(string, test, flag, cond=None, prefix="NO", addtxt="", skip=F cond = test if test is not None: if cond: - string = "{} {}".format(string, flag) + string = f"{string} {flag}" else: - string = "{0} {2}{1}".format(string, flag, prefix) + string = f"{string} {prefix}{flag}" if addtxt: - string = "{} {}".format(string, addtxt) + string = f"{string} {addtxt}" return string @@ -1224,7 +1220,7 @@ def _verify_password(role, password, verifier, method): def _md5_password(role, password): return "md5{}".format( hashlib.md5( # nosec - salt.utils.stringutils.to_bytes("{}{}".format(password, role)) + salt.utils.stringutils.to_bytes(f"{password}{role}") ).hexdigest() ) @@ -1343,7 +1339,7 @@ def _role_cmd_args( if isinstance(groups, list): groups = ",".join(groups) for group in groups.split(","): - sub_cmd = '{}; GRANT "{}" TO "{}"'.format(sub_cmd, group, name) + sub_cmd = f'{sub_cmd}; GRANT "{group}" TO "{name}"' return sub_cmd @@ -1380,7 +1376,7 @@ def _role_create( log.info("%s '%s' already exists", typ_.capitalize(), name) return False - sub_cmd = 'CREATE ROLE "{}" WITH'.format(name) + sub_cmd = f'CREATE ROLE "{name}" WITH' sub_cmd = "{} {}".format( sub_cmd, _role_cmd_args( @@ -1506,7 +1502,7 @@ def _role_update( log.info("%s '%s' could not be found", typ_.capitalize(), name) return False - sub_cmd = 'ALTER ROLE "{}" WITH'.format(name) + sub_cmd = f'ALTER ROLE "{name}" WITH' sub_cmd = "{} {}".format( sub_cmd, _role_cmd_args( @@ -1613,7 +1609,7 @@ def _role_remove( return False # user exists, proceed - sub_cmd = 'DROP ROLE "{}"'.format(name) + sub_cmd = f'DROP ROLE "{name}"' _psql_prepare_and_run( ["-c", sub_cmd], runas=runas, @@ -1995,14 +1991,14 @@ def create_extension( args = ["CREATE EXTENSION"] if if_not_exists: args.append("IF NOT EXISTS") - args.append('"{}"'.format(name)) + args.append(f'"{name}"') sargs = [] if schema: - sargs.append('SCHEMA "{}"'.format(schema)) + sargs.append(f'SCHEMA "{schema}"') if ext_version: - sargs.append("VERSION {}".format(ext_version)) + sargs.append(f"VERSION {ext_version}") if from_version: - sargs.append("FROM {}".format(from_version)) + sargs.append(f"FROM {from_version}") if sargs: args.append("WITH") args.extend(sargs) @@ -2011,13 +2007,9 @@ def create_extension( else: args = [] if schema and _EXTENSION_TO_MOVE in mtdata: - args.append( - 'ALTER EXTENSION "{}" SET SCHEMA "{}";'.format(name, schema) - ) + args.append(f'ALTER EXTENSION "{name}" SET SCHEMA "{schema}";') if ext_version and _EXTENSION_TO_UPGRADE in mtdata: - args.append( - 'ALTER EXTENSION "{}" UPDATE TO {};'.format(name, ext_version) - ) + args.append(f'ALTER EXTENSION "{name}" UPDATE TO {ext_version};') cmd = " ".join(args).strip() if cmd: _psql_prepare_and_run( @@ -2227,7 +2219,7 @@ def owner_to( sqlfile = tempfile.NamedTemporaryFile() sqlfile.write("begin;\n") - sqlfile.write('alter database "{}" owner to "{}";\n'.format(dbname, ownername)) + sqlfile.write(f'alter database "{dbname}" owner to "{ownername}";\n') queries = ( # schemas @@ -2335,9 +2327,9 @@ def schema_create( log.info("'%s' already exists in '%s'", name, dbname) return False - sub_cmd = 'CREATE SCHEMA "{}"'.format(name) + sub_cmd = f'CREATE SCHEMA "{name}"' if owner is not None: - sub_cmd = '{} AUTHORIZATION "{}"'.format(sub_cmd, owner) + sub_cmd = f'{sub_cmd} AUTHORIZATION "{owner}"' ret = _psql_prepare_and_run( ["-c", sub_cmd], @@ -2401,7 +2393,7 @@ def schema_remove( return False # schema exists, proceed - sub_cmd = 'DROP SCHEMA "{}"'.format(name) + sub_cmd = f'DROP SCHEMA "{name}"' _psql_prepare_and_run( ["-c", sub_cmd], runas=user, @@ -2721,7 +2713,7 @@ def language_create( log.info("Language %s already exists in %s", name, maintenance_db) return False - query = "CREATE LANGUAGE {}".format(name) + query = f"CREATE LANGUAGE {name}" ret = _psql_prepare_and_run( ["-c", query], @@ -2776,7 +2768,7 @@ def language_remove( log.info("Language %s does not exist in %s", name, maintenance_db) return False - query = "DROP LANGUAGE {}".format(name) + query = f"DROP LANGUAGE {name}" ret = _psql_prepare_and_run( ["-c", query], @@ -3035,9 +3027,7 @@ def _validate_privileges(object_type, privs, privileges): _perms.append("ALL") if object_type not in _PRIVILEGES_OBJECTS: - raise SaltInvocationError( - "Invalid object_type: {} provided".format(object_type) - ) + raise SaltInvocationError(f"Invalid object_type: {object_type} provided") if not set(privs).issubset(set(_perms)): raise SaltInvocationError( @@ -3145,9 +3135,7 @@ def privileges_list( query = _make_privileges_list_query(name, object_type, prepend) if object_type not in _PRIVILEGES_OBJECTS: - raise SaltInvocationError( - "Invalid object_type: {} provided".format(object_type) - ) + raise SaltInvocationError(f"Invalid object_type: {object_type} provided") rows = psql_query( query, @@ -3439,15 +3427,15 @@ def privileges_grant( _grants = ",".join(_privs) if object_type in ["table", "sequence"]: - on_part = '{}."{}"'.format(prepend, object_name) + on_part = f'{prepend}."{object_name}"' elif object_type == "function": - on_part = "{}".format(object_name) + on_part = f"{object_name}" else: - on_part = '"{}"'.format(object_name) + on_part = f'"{object_name}"' if grant_option: if object_type == "group": - query = 'GRANT {} TO "{}" WITH ADMIN OPTION'.format(object_name, name) + query = f'GRANT {object_name} TO "{name}" WITH ADMIN OPTION' elif object_type in ("table", "sequence") and object_name.upper() == "ALL": query = 'GRANT {} ON ALL {}S IN SCHEMA {} TO "{}" WITH GRANT OPTION'.format( _grants, object_type.upper(), prepend, name @@ -3458,7 +3446,7 @@ def privileges_grant( ) else: if object_type == "group": - query = 'GRANT {} TO "{}"'.format(object_name, name) + query = f'GRANT {object_name} TO "{name}"' elif object_type in ("table", "sequence") and object_name.upper() == "ALL": query = 'GRANT {} ON ALL {}S IN SCHEMA {} TO "{}"'.format( _grants, object_type.upper(), prepend, name @@ -3587,12 +3575,12 @@ def privileges_revoke( _grants = ",".join(_privs) if object_type in ["table", "sequence"]: - on_part = "{}.{}".format(prepend, object_name) + on_part = f"{prepend}.{object_name}" else: on_part = object_name if object_type == "group": - query = "REVOKE {} FROM {}".format(object_name, name) + query = f"REVOKE {object_name} FROM {name}" else: query = "REVOKE {} ON {} {} FROM {}".format( _grants, object_type.upper(), on_part, name diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index 841811dcdfe..a0843130593 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -10,8 +10,8 @@ import hashlib import logging import multiprocessing import os -import pipes import re +import shlex import shutil import socket import stat @@ -199,7 +199,7 @@ def __ssh_gateway_arguments(kwargs): "-oUserKnownHostsFile=/dev/null", "-oControlPath=none", str(ssh_gateway_key), - "{}@{}".format(ssh_gateway_user, ssh_gateway), + f"{ssh_gateway_user}@{ssh_gateway}", "-p", str(ssh_gateway_port), str(ssh_gateway_command), @@ -228,18 +228,18 @@ def os_script(os_, vm_=None, opts=None, minion=""): # The user provided an absolute path to the deploy script, let's use it return __render_script(os_, vm_, opts, minion) - if os.path.isabs("{}.sh".format(os_)): + if os.path.isabs(f"{os_}.sh"): # The user provided an absolute path to the deploy script, although no # extension was provided. Let's use it anyway. - return __render_script("{}.sh".format(os_), vm_, opts, minion) + return __render_script(f"{os_}.sh", vm_, opts, minion) for search_path in opts["deploy_scripts_search_path"]: if os.path.isfile(os.path.join(search_path, os_)): return __render_script(os.path.join(search_path, os_), vm_, opts, minion) - if os.path.isfile(os.path.join(search_path, "{}.sh".format(os_))): + if os.path.isfile(os.path.join(search_path, f"{os_}.sh")): return __render_script( - os.path.join(search_path, "{}.sh".format(os_)), vm_, opts, minion + os.path.join(search_path, f"{os_}.sh"), vm_, opts, minion ) # No deploy script was found, return an empty string return "" @@ -416,7 +416,7 @@ def bootstrap(vm_, opts=None): ) if key_filename is not None and not os.path.isfile(key_filename): raise SaltCloudConfigError( - "The defined ssh_keyfile '{}' does not exist".format(key_filename) + f"The defined ssh_keyfile '{key_filename}' does not exist" ) has_ssh_agent = False if ( @@ -782,8 +782,8 @@ def wait_for_port( # Don't add new hosts to the host key database "-oStrictHostKeyChecking=no", # make sure ssh can time out on connection lose - "-oServerAliveInterval={}".format(server_alive_interval), - "-oServerAliveCountMax={}".format(server_alive_count_max), + f"-oServerAliveInterval={server_alive_interval}", + f"-oServerAliveCountMax={server_alive_count_max}", # Set hosts key database path to /dev/null, i.e., non-existing "-oUserKnownHostsFile=/dev/null", # Don't re-use the SSH connection. Less failures. @@ -808,21 +808,21 @@ def wait_for_port( ] ) # Netcat command testing remote port - command = "nc -z -w5 -q0 {} {}".format(host, port) + command = f"nc -z -w5 -q0 {host} {port}" # SSH command pcmd = "ssh {} {}@{} -p {} {}".format( " ".join(ssh_args), gateway["ssh_gateway_user"], ssh_gateway, ssh_gateway_port, - pipes.quote("date"), + shlex.quote("date"), ) cmd = "ssh {} {}@{} -p {} {}".format( " ".join(ssh_args), gateway["ssh_gateway_user"], ssh_gateway, ssh_gateway_port, - pipes.quote(command), + shlex.quote(command), ) log.debug("SSH command: '%s'", cmd) @@ -893,7 +893,7 @@ class Client: service_name=None, ): self.service_name = service_name - self._exe_file = "{}.exe".format(self.service_name) + self._exe_file = f"{self.service_name}.exe" self._client = PsExecClient(server, username, password, port, encrypt) self._client._service = ScmrService(self.service_name, self._client.session) @@ -943,7 +943,7 @@ class Client: # delete the PAExec executable smb_tree = TreeConnect( self._client.session, - r"\\{}\ADMIN$".format(self._client.connection.server_name), + rf"\\{self._client.connection.server_name}\ADMIN$", ) log.info("Connecting to SMB Tree %s", smb_tree.share_name) smb_tree.connect() @@ -968,10 +968,10 @@ def run_winexe_command(cmd, args, host, username, password, port=445): """ Run a command remotely via the winexe executable """ - creds = "-U '{}%{}' //{}".format(username, password, host) - logging_creds = "-U '{}%XXX-REDACTED-XXX' //{}".format(username, host) - cmd = "winexe {} {} {}".format(creds, cmd, args) - logging_cmd = "winexe {} {} {}".format(logging_creds, cmd, args) + creds = f"-U '{username}%{password}' //{host}" + logging_creds = f"-U '{username}%XXX-REDACTED-XXX' //{host}" + cmd = f"winexe {creds} {cmd} {args}" + logging_cmd = f"winexe {logging_creds} {cmd} {args}" return win_cmd(cmd, logging_command=logging_cmd) @@ -979,7 +979,7 @@ def run_psexec_command(cmd, args, host, username, password, port=445): """ Run a command remotely using the psexec protocol """ - service_name = "PS-Exec-{}".format(uuid.uuid4()) + service_name = f"PS-Exec-{uuid.uuid4()}" with Client( host, username, password, port=port, encrypt=False, service_name=service_name ) as client: @@ -1098,7 +1098,7 @@ def validate_windows_cred_winexe( """ Check if the windows credentials are valid """ - cmd = "winexe -U '{}%{}' //{} \"hostname\"".format(username, password, host) + cmd = f"winexe -U '{username}%{password}' //{host} \"hostname\"" logging_cmd = "winexe -U '{}%XXX-REDACTED-XXX' //{} \"hostname\"".format( username, host ) @@ -1240,7 +1240,7 @@ def deploy_windows( winrm_port=5986, winrm_use_ssl=True, winrm_verify_ssl=True, - **kwargs + **kwargs, ): """ Copy the install files to a remote Windows box, and execute them @@ -1299,20 +1299,20 @@ def deploy_windows( salt.utils.smb.mkdirs("salttemp", conn=smb_conn) root_dir = "ProgramData/Salt Project/Salt" - salt.utils.smb.mkdirs("{}/conf/pki/minion".format(root_dir), conn=smb_conn) + salt.utils.smb.mkdirs(f"{root_dir}/conf/pki/minion", conn=smb_conn) root_dir = "ProgramData\\Salt Project\\Salt" if minion_pub: salt.utils.smb.put_str( minion_pub, - "{}\\conf\\pki\\minion\\minion.pub".format(root_dir), + f"{root_dir}\\conf\\pki\\minion\\minion.pub", conn=smb_conn, ) if minion_pem: salt.utils.smb.put_str( minion_pem, - "{}\\conf\\pki\\minion\\minion.pem".format(root_dir), + f"{root_dir}\\conf\\pki\\minion\\minion.pem", conn=smb_conn, ) @@ -1324,7 +1324,7 @@ def deploy_windows( try: salt.utils.smb.put_file( master_sign_pub_file, - "{}\\conf\\pki\\minion\\master_sign.pub".format(root_dir), + f"{root_dir}\\conf\\pki\\minion\\master_sign.pub", conn=smb_conn, ) except Exception as e: # pylint: disable=broad-except @@ -1342,16 +1342,16 @@ def deploy_windows( installer = comps[-1] salt.utils.smb.put_file( win_installer, - "salttemp\\{}".format(installer), + f"salttemp\\{installer}", "C$", conn=smb_conn, ) - cmd = "c:\\salttemp\\{}".format(installer) + cmd = f"c:\\salttemp\\{installer}" args = [ "/S", - "/master={}".format(_format_master_param(master)), - "/minion-name={}".format(name), + f"/master={_format_master_param(master)}", + f"/minion-name={name}", ] if use_winrm: @@ -1362,7 +1362,7 @@ def deploy_windows( ) if ret_code != 0: - raise Exception("Fail installer {}".format(ret_code)) + raise Exception(f"Fail installer {ret_code}") # Copy over minion_conf if minion_conf: @@ -1378,7 +1378,7 @@ def deploy_windows( if minion_grains: salt.utils.smb.put_str( salt_config_to_yaml(minion_grains, line_break="\r\n"), - "{}\\conf\\grains".format(root_dir), + f"{root_dir}\\conf\\grains", conn=smb_conn, ) # Add special windows minion configuration @@ -1395,7 +1395,7 @@ def deploy_windows( minion_conf = dict(minion_conf, **windows_minion_conf) salt.utils.smb.put_str( salt_config_to_yaml(minion_conf, line_break="\r\n"), - "{}\\conf\\minion".format(root_dir), + f"{root_dir}\\conf\\minion", conn=smb_conn, ) # Delete C:\salttmp\ and installer file @@ -1405,7 +1405,7 @@ def deploy_windows( winrm_cmd(winrm_session, "rmdir", ["/Q", "/S", "C:\\salttemp\\"]) else: salt.utils.smb.delete_file( - "salttemp\\{}".format(installer), "C$", conn=smb_conn + f"salttemp\\{installer}", "C$", conn=smb_conn ) salt.utils.smb.delete_directory("salttemp", "C$", conn=smb_conn) # Shell out to psexec to ensure salt-minion service started @@ -1429,8 +1429,8 @@ def deploy_windows( # Fire deploy action fire_event( "event", - "{} has been deployed at {}".format(name, host), - "salt/cloud/{}/deploy_windows".format(name), + f"{name} has been deployed at {host}", + f"salt/cloud/{name}/deploy_windows", args={"name": name}, sock_dir=opts.get("sock_dir", os.path.join(__opts__["sock_dir"], "master")), transport=opts.get("transport", "zeromq"), @@ -1480,7 +1480,7 @@ def deploy_script( master_sign_pub_file=None, cloud_grains=None, force_minion_config=False, - **kwargs + **kwargs, ): """ Copy a deploy script to a remote server, execute it, and remove it @@ -1496,7 +1496,7 @@ def deploy_script( ) if key_filename is not None and not os.path.isfile(key_filename): raise SaltCloudConfigError( - "The defined key_filename '{}' does not exist".format(key_filename) + f"The defined key_filename '{key_filename}' does not exist" ) gateway = None @@ -1543,35 +1543,28 @@ def deploy_script( ssh_kwargs["password"] = password if root_cmd( - "test -e '{}'".format(tmp_dir), - tty, - sudo, - allow_failure=True, - **ssh_kwargs + f"test -e '{tmp_dir}'", tty, sudo, allow_failure=True, **ssh_kwargs ): ret = root_cmd( - "sh -c \"( mkdir -p -m 700 '{}' )\"".format(tmp_dir), + f"sh -c \"( mkdir -p -m 700 '{tmp_dir}' )\"", tty, sudo, - **ssh_kwargs + **ssh_kwargs, ) if ret: raise SaltCloudSystemExit( - "Can't create temporary directory in {} !".format(tmp_dir) + f"Can't create temporary directory in {tmp_dir} !" ) if sudo: comps = tmp_dir.lstrip("/").rstrip("/").split("/") if comps: if len(comps) > 1 or comps[0] != "tmp": ret = root_cmd( - 'chown {} "{}"'.format(username, tmp_dir), - tty, - sudo, - **ssh_kwargs + f'chown {username} "{tmp_dir}"', tty, sudo, **ssh_kwargs ) if ret: raise SaltCloudSystemExit( - "Cant set {} ownership on {}".format(username, tmp_dir) + f"Cant set {username} ownership on {tmp_dir}" ) if not isinstance(file_map, dict): @@ -1601,15 +1594,13 @@ def deploy_script( remote_dir = os.path.dirname(remote_file) if remote_dir not in remote_dirs: - root_cmd( - "mkdir -p '{}'".format(remote_dir), tty, sudo, **ssh_kwargs - ) + root_cmd(f"mkdir -p '{remote_dir}'", tty, sudo, **ssh_kwargs) if ssh_kwargs["username"] != "root": root_cmd( "chown {} '{}'".format(ssh_kwargs["username"], remote_dir), tty, sudo, - **ssh_kwargs + **ssh_kwargs, ) remote_dirs.append(remote_dir) ssh_file(opts, remote_file, kwargs=ssh_kwargs, local_file=local_file) @@ -1617,21 +1608,21 @@ def deploy_script( # Minion configuration if minion_pem: - ssh_file(opts, "{}/minion.pem".format(tmp_dir), minion_pem, ssh_kwargs) + ssh_file(opts, f"{tmp_dir}/minion.pem", minion_pem, ssh_kwargs) ret = root_cmd( - "chmod 600 '{}/minion.pem'".format(tmp_dir), tty, sudo, **ssh_kwargs + f"chmod 600 '{tmp_dir}/minion.pem'", tty, sudo, **ssh_kwargs ) if ret: raise SaltCloudSystemExit( - "Can't set perms on {}/minion.pem".format(tmp_dir) + f"Can't set perms on {tmp_dir}/minion.pem" ) if minion_pub: - ssh_file(opts, "{}/minion.pub".format(tmp_dir), minion_pub, ssh_kwargs) + ssh_file(opts, f"{tmp_dir}/minion.pub", minion_pub, ssh_kwargs) if master_sign_pub_file: ssh_file( opts, - "{}/master_sign.pub".format(tmp_dir), + f"{tmp_dir}/master_sign.pub", kwargs=ssh_kwargs, local_file=master_sign_pub_file, ) @@ -1649,7 +1640,7 @@ def deploy_script( if minion_grains: ssh_file( opts, - "{}/grains".format(tmp_dir), + f"{tmp_dir}/grains", salt_config_to_yaml(minion_grains), ssh_kwargs, ) @@ -1657,24 +1648,22 @@ def deploy_script( minion_conf["grains"] = {"salt-cloud": cloud_grains} ssh_file( opts, - "{}/minion".format(tmp_dir), + f"{tmp_dir}/minion", salt_config_to_yaml(minion_conf), ssh_kwargs, ) # Master configuration if master_pem: - ssh_file(opts, "{}/master.pem".format(tmp_dir), master_pem, ssh_kwargs) + ssh_file(opts, f"{tmp_dir}/master.pem", master_pem, ssh_kwargs) ret = root_cmd( - "chmod 600 '{}/master.pem'".format(tmp_dir), tty, sudo, **ssh_kwargs + f"chmod 600 '{tmp_dir}/master.pem'", tty, sudo, **ssh_kwargs ) if ret: - raise SaltCloudSystemExit( - "Cant set perms on {}/master.pem".format(tmp_dir) - ) + raise SaltCloudSystemExit(f"Cant set perms on {tmp_dir}/master.pem") if master_pub: - ssh_file(opts, "{}/master.pub".format(tmp_dir), master_pub, ssh_kwargs) + ssh_file(opts, f"{tmp_dir}/master.pub", master_pub, ssh_kwargs) if master_conf: if not isinstance(master_conf, dict): @@ -1688,34 +1677,31 @@ def deploy_script( ssh_file( opts, - "{}/master".format(tmp_dir), + f"{tmp_dir}/master", salt_config_to_yaml(master_conf), ssh_kwargs, ) # XXX: We need to make these paths configurable - preseed_minion_keys_tempdir = "{}/preseed-minion-keys".format(tmp_dir) + preseed_minion_keys_tempdir = f"{tmp_dir}/preseed-minion-keys" if preseed_minion_keys is not None: # Create remote temp dir ret = root_cmd( - "mkdir '{}'".format(preseed_minion_keys_tempdir), - tty, - sudo, - **ssh_kwargs + f"mkdir '{preseed_minion_keys_tempdir}'", tty, sudo, **ssh_kwargs ) if ret: raise SaltCloudSystemExit( - "Cant create {}".format(preseed_minion_keys_tempdir) + f"Cant create {preseed_minion_keys_tempdir}" ) ret = root_cmd( - "chmod 700 '{}'".format(preseed_minion_keys_tempdir), + f"chmod 700 '{preseed_minion_keys_tempdir}'", tty, sudo, - **ssh_kwargs + **ssh_kwargs, ) if ret: raise SaltCloudSystemExit( - "Can't set perms on {}".format(preseed_minion_keys_tempdir) + f"Can't set perms on {preseed_minion_keys_tempdir}" ) if ssh_kwargs["username"] != "root": root_cmd( @@ -1724,7 +1710,7 @@ def deploy_script( ), tty, sudo, - **ssh_kwargs + **ssh_kwargs, ) # Copy pre-seed minion keys @@ -1734,10 +1720,10 @@ def deploy_script( if ssh_kwargs["username"] != "root": root_cmd( - "chown -R root '{}'".format(preseed_minion_keys_tempdir), + f"chown -R root '{preseed_minion_keys_tempdir}'", tty, sudo, - **ssh_kwargs + **ssh_kwargs, ) if ret: raise SaltCloudSystemExit( @@ -1751,25 +1737,21 @@ def deploy_script( for command in preflight_cmds: cmd_ret = root_cmd(command, tty, sudo, **ssh_kwargs) if cmd_ret: - raise SaltCloudSystemExit( - "Pre-flight command failed: '{}'".format(command) - ) + raise SaltCloudSystemExit(f"Pre-flight command failed: '{command}'") # The actual deploy script if script: # got strange escaping issues with sudoer, going onto a # subshell fixes that - ssh_file(opts, "{}/deploy.sh".format(tmp_dir), script, ssh_kwargs) + ssh_file(opts, f"{tmp_dir}/deploy.sh", script, ssh_kwargs) ret = root_cmd( - "sh -c \"( chmod +x '{}/deploy.sh' )\";exit $?".format(tmp_dir), + f"sh -c \"( chmod +x '{tmp_dir}/deploy.sh' )\";exit $?", tty, sudo, - **ssh_kwargs + **ssh_kwargs, ) if ret: - raise SaltCloudSystemExit( - "Can't set perms on {}/deploy.sh".format(tmp_dir) - ) + raise SaltCloudSystemExit(f"Can't set perms on {tmp_dir}/deploy.sh") time_used = time.mktime(time.localtime()) - time.mktime(starttime) newtimeout = timeout - time_used @@ -1785,7 +1767,7 @@ def deploy_script( kwargs=dict( name=name, sock_dir=sock_dir, timeout=newtimeout, queue=queue ), - name="DeployScriptCheckAuth({})".format(name), + name=f"DeployScriptCheckAuth({name})", ) log.debug("Starting new process to wait for salt-minion") process.start() @@ -1793,7 +1775,7 @@ def deploy_script( # Run the deploy script if script: if "bootstrap-salt" in script: - deploy_command += " -c '{}'".format(tmp_dir) + deploy_command += f" -c '{tmp_dir}'" if force_minion_config: deploy_command += " -F" if make_syndic is True: @@ -1805,9 +1787,9 @@ def deploy_script( if keep_tmp is True: deploy_command += " -K" if preseed_minion_keys is not None: - deploy_command += " -k '{}'".format(preseed_minion_keys_tempdir) + deploy_command += f" -k '{preseed_minion_keys_tempdir}'" if script_args: - deploy_command += " {}".format(script_args) + deploy_command += f" {script_args}" if script_env: if not isinstance(script_env, dict): @@ -1826,15 +1808,15 @@ def deploy_script( # Upload our environ setter wrapper ssh_file( opts, - "{}/environ-deploy-wrapper.sh".format(tmp_dir), + f"{tmp_dir}/environ-deploy-wrapper.sh", "\n".join(environ_script_contents), ssh_kwargs, ) root_cmd( - "chmod +x '{}/environ-deploy-wrapper.sh'".format(tmp_dir), + f"chmod +x '{tmp_dir}/environ-deploy-wrapper.sh'", tty, sudo, - **ssh_kwargs + **ssh_kwargs, ) # The deploy command is now our wrapper deploy_command = "'{}/environ-deploy-wrapper.sh'".format( @@ -1842,22 +1824,20 @@ def deploy_script( ) if root_cmd(deploy_command, tty, sudo, **ssh_kwargs) != 0: raise SaltCloudSystemExit( - "Executing the command '{}' failed".format(deploy_command) + f"Executing the command '{deploy_command}' failed" ) log.debug("Executed command '%s'", deploy_command) # Remove the deploy script if not keep_tmp: - root_cmd( - "rm -f '{}/deploy.sh'".format(tmp_dir), tty, sudo, **ssh_kwargs - ) + root_cmd(f"rm -f '{tmp_dir}/deploy.sh'", tty, sudo, **ssh_kwargs) log.debug("Removed %s/deploy.sh", tmp_dir) if script_env: root_cmd( - "rm -f '{}/environ-deploy-wrapper.sh'".format(tmp_dir), + f"rm -f '{tmp_dir}/environ-deploy-wrapper.sh'", tty, sudo, - **ssh_kwargs + **ssh_kwargs, ) log.debug("Removed %s/environ-deploy-wrapper.sh", tmp_dir) @@ -1866,57 +1846,40 @@ def deploy_script( else: # Remove minion configuration if minion_pub: - root_cmd( - "rm -f '{}/minion.pub'".format(tmp_dir), tty, sudo, **ssh_kwargs - ) + root_cmd(f"rm -f '{tmp_dir}/minion.pub'", tty, sudo, **ssh_kwargs) log.debug("Removed %s/minion.pub", tmp_dir) if minion_pem: - root_cmd( - "rm -f '{}/minion.pem'".format(tmp_dir), tty, sudo, **ssh_kwargs - ) + root_cmd(f"rm -f '{tmp_dir}/minion.pem'", tty, sudo, **ssh_kwargs) log.debug("Removed %s/minion.pem", tmp_dir) if minion_conf: - root_cmd( - "rm -f '{}/grains'".format(tmp_dir), tty, sudo, **ssh_kwargs - ) + root_cmd(f"rm -f '{tmp_dir}/grains'", tty, sudo, **ssh_kwargs) log.debug("Removed %s/grains", tmp_dir) - root_cmd( - "rm -f '{}/minion'".format(tmp_dir), tty, sudo, **ssh_kwargs - ) + root_cmd(f"rm -f '{tmp_dir}/minion'", tty, sudo, **ssh_kwargs) log.debug("Removed %s/minion", tmp_dir) if master_sign_pub_file: root_cmd( - "rm -f {}/master_sign.pub".format(tmp_dir), - tty, - sudo, - **ssh_kwargs + f"rm -f {tmp_dir}/master_sign.pub", tty, sudo, **ssh_kwargs ) log.debug("Removed %s/master_sign.pub", tmp_dir) # Remove master configuration if master_pub: - root_cmd( - "rm -f '{}/master.pub'".format(tmp_dir), tty, sudo, **ssh_kwargs - ) + root_cmd(f"rm -f '{tmp_dir}/master.pub'", tty, sudo, **ssh_kwargs) log.debug("Removed %s/master.pub", tmp_dir) if master_pem: - root_cmd( - "rm -f '{}/master.pem'".format(tmp_dir), tty, sudo, **ssh_kwargs - ) + root_cmd(f"rm -f '{tmp_dir}/master.pem'", tty, sudo, **ssh_kwargs) log.debug("Removed %s/master.pem", tmp_dir) if master_conf: - root_cmd( - "rm -f '{}/master'".format(tmp_dir), tty, sudo, **ssh_kwargs - ) + root_cmd(f"rm -f '{tmp_dir}/master'", tty, sudo, **ssh_kwargs) log.debug("Removed %s/master", tmp_dir) # Remove pre-seed keys directory if preseed_minion_keys is not None: root_cmd( - "rm -rf '{}'".format(preseed_minion_keys_tempdir), + f"rm -rf '{preseed_minion_keys_tempdir}'", tty, sudo, - **ssh_kwargs + **ssh_kwargs, ) log.debug("Removed %s", preseed_minion_keys_tempdir) @@ -1931,15 +1894,13 @@ def deploy_script( # for line in output: # print(line) log.info("Executing %s on the salt-minion", start_action) - root_cmd( - "salt-call {}".format(start_action), tty, sudo, **ssh_kwargs - ) + root_cmd(f"salt-call {start_action}", tty, sudo, **ssh_kwargs) log.info("Finished executing %s on the salt-minion", start_action) # Fire deploy action fire_event( "event", - "{} has been deployed at {}".format(name, host), - "salt/cloud/{}/deploy_script".format(name), + f"{name} has been deployed at {host}", + f"salt/cloud/{name}/deploy_script", args={"name": name, "host": host}, sock_dir=opts.get( "sock_dir", os.path.join(__opts__["sock_dir"], "master") @@ -1972,7 +1933,7 @@ def run_inline_script( tty=None, opts=None, tmp_dir="/tmp/.saltcloud-inline_script", - **kwargs + **kwargs, ): """ Run the inline script commands, one by one @@ -2029,11 +1990,11 @@ def run_inline_script( # TODO: check edge cases (e.g. ssh gateways, salt deploy disabled, etc.) if ( root_cmd( - 'test -e \\"{}\\"'.format(tmp_dir), + f'test -e \\"{tmp_dir}\\"', tty, sudo, allow_failure=True, - **ssh_kwargs + **ssh_kwargs, ) and inline_script ): @@ -2041,11 +2002,11 @@ def run_inline_script( for cmd_line in inline_script: log.info("Executing inline command: %s", cmd_line) ret = root_cmd( - 'sh -c "( {} )"'.format(cmd_line), + f'sh -c "( {cmd_line} )"', tty, sudo, allow_failure=True, - **ssh_kwargs + **ssh_kwargs, ) if ret: log.info("[%s] Output: %s", cmd_line, ret) @@ -2149,7 +2110,7 @@ def _exec_ssh_cmd(cmd, error_msg=None, allow_failure=False, **kwargs): time.sleep(0.5) if proc.exitstatus != 0 and allow_failure is False: raise SaltCloudSystemExit( - "Command '{}' failed. Exit code: {}".format(cmd, proc.exitstatus) + f"Command '{cmd}' failed. Exit code: {proc.exitstatus}" ) return proc.exitstatus except salt.utils.vt.TerminalException as err: @@ -2252,7 +2213,7 @@ def scp_file(dest_path, contents=None, kwargs=None, local_file=None): cmd, error_msg="Failed to upload file '{0}': {1}\n{2}", password_retries=3, - **kwargs + **kwargs, ) finally: if contents is not None: @@ -2370,7 +2331,7 @@ def sftp_file(dest_path, contents=None, kwargs=None, local_file=None): cmd, error_msg="Failed to upload file '{0}': {1}\n{2}", password_retries=3, - **kwargs + **kwargs, ) finally: if contents is not None: @@ -2430,11 +2391,11 @@ def root_cmd(command, tty, sudo, allow_failure=False, **kwargs): if sudo: if sudo_password is None: - command = "sudo {}".format(command) + command = f"sudo {command}" logging_command = command else: - logging_command = 'sudo -S "XXX-REDACTED-XXX" {}'.format(command) - command = "sudo -S {}".format(command) + logging_command = f'sudo -S "XXX-REDACTED-XXX" {command}' + command = f"sudo -S {command}" log.debug("Using sudo to run command %s", logging_command) @@ -2453,9 +2414,9 @@ def root_cmd(command, tty, sudo, allow_failure=False, **kwargs): ssh_args.extend( [ # Don't add new hosts to the host key database - "-oStrictHostKeyChecking={}".format(host_key_checking), + f"-oStrictHostKeyChecking={host_key_checking}", # Set hosts key database path to /dev/null, i.e., non-existing - "-oUserKnownHostsFile={}".format(known_hosts_file), + f"-oUserKnownHostsFile={known_hosts_file}", # Don't re-use the SSH connection. Less failures. "-oControlPath=none", ] @@ -2488,12 +2449,12 @@ def root_cmd(command, tty, sudo, allow_failure=False, **kwargs): cmd = "ssh {0} {1[username]}@{1[hostname]} ".format(" ".join(ssh_args), kwargs) logging_command = cmd + logging_command - cmd = cmd + pipes.quote(command) + cmd = cmd + shlex.quote(command) hard_timeout = kwargs.get("hard_timeout") if hard_timeout is not None: - logging_command = "timeout {} {}".format(hard_timeout, logging_command) - cmd = "timeout {} {}".format(hard_timeout, cmd) + logging_command = f"timeout {hard_timeout} {logging_command}" + cmd = f"timeout {hard_timeout} {cmd}" log.debug("SSH command: '%s'", logging_command) @@ -2515,7 +2476,7 @@ def check_auth(name, sock_dir=None, queue=None, timeout=300): ret = event.get_event(full=True) if ret is None: continue - if ret["tag"] == "salt/minion/{}/start".format(name): + if ret["tag"] == f"salt/minion/{name}/start": queue.put(name) newtimeout = 0 log.debug("Minion %s is ready to receive commands", name) @@ -2561,7 +2522,7 @@ def check_name(name, safe_chars): """ Check whether the specified name contains invalid characters """ - regexp = re.compile("[^{}]".format(safe_chars)) + regexp = re.compile(f"[^{safe_chars}]") if regexp.search(name): raise SaltCloudException( "{} contains characters not supported by this cloud provider. " @@ -2855,7 +2816,7 @@ def request_minion_cachedir( "provider": provider, } - fname = "{}.p".format(minion_id) + fname = f"{minion_id}.p" path = os.path.join(base, "requested", fname) with salt.utils.files.fopen(path, "wb") as fh_: salt.utils.msgpack.dump(data, fh_, encoding=MSGPACK_ENCODING) @@ -2886,7 +2847,7 @@ def change_minion_cachedir( if base is None: base = __opts__["cachedir"] - fname = "{}.p".format(minion_id) + fname = f"{minion_id}.p" path = os.path.join(base, cachedir, fname) with salt.utils.files.fopen(path, "r") as fh_: @@ -2909,7 +2870,7 @@ def activate_minion_cachedir(minion_id, base=None): if base is None: base = __opts__["cachedir"] - fname = "{}.p".format(minion_id) + fname = f"{minion_id}.p" src = os.path.join(base, "requested", fname) dst = os.path.join(base, "active") shutil.move(src, dst) @@ -2931,7 +2892,7 @@ def delete_minion_cachedir(minion_id, provider, opts, base=None): base = __opts__["cachedir"] driver = next(iter(__opts__["providers"][provider].keys())) - fname = "{}.p".format(minion_id) + fname = f"{minion_id}.p" for cachedir in "requested", "active": path = os.path.join(base, cachedir, driver, provider, fname) log.debug("path: %s", path) @@ -3024,7 +2985,7 @@ def update_bootstrap(config, url=None): # in last case, assuming we got a script content else: script_content = url - script_name = "{}.sh".format(hashlib.sha1(script_content).hexdigest()) + script_name = f"{hashlib.sha1(script_content).hexdigest()}.sh" if not script_content: raise ValueError("No content in bootstrap script !") @@ -3118,7 +3079,7 @@ def cache_node_list(nodes, provider, opts): for node in nodes: diff_node_cache(prov_dir, node, nodes[node], opts) - path = os.path.join(prov_dir, "{}.p".format(node)) + path = os.path.join(prov_dir, f"{node}.p") with salt.utils.files.fopen(path, "wb") as fh_: salt.utils.msgpack.dump(nodes[node], fh_, encoding=MSGPACK_ENCODING) @@ -3173,7 +3134,7 @@ def missing_node_cache(prov_dir, node_list, provider, opts): fire_event( "event", "cached node missing from provider", - "salt/cloud/{}/cache_node_missing".format(node), + f"salt/cloud/{node}/cache_node_missing", args={"missing node": node}, sock_dir=opts.get( "sock_dir", os.path.join(__opts__["sock_dir"], "master") @@ -3201,7 +3162,7 @@ def diff_node_cache(prov_dir, node, new_data, opts): if node is None: return - path = "{}.p".format(os.path.join(prov_dir, node)) + path = f"{os.path.join(prov_dir, node)}.p" if not os.path.exists(path): event_data = _strip_cache_events(new_data, opts) @@ -3209,7 +3170,7 @@ def diff_node_cache(prov_dir, node, new_data, opts): fire_event( "event", "new node found", - "salt/cloud/{}/cache_node_new".format(node), + f"salt/cloud/{node}/cache_node_new", args={"new_data": event_data}, sock_dir=opts.get("sock_dir", os.path.join(__opts__["sock_dir"], "master")), transport=opts.get("transport", "zeromq"), @@ -3233,7 +3194,7 @@ def diff_node_cache(prov_dir, node, new_data, opts): fire_event( "event", "node data differs", - "salt/cloud/{}/cache_node_diff".format(node), + f"salt/cloud/{node}/cache_node_diff", args={ "new_data": _strip_cache_events(new_data, opts), "cache_data": _strip_cache_events(cache_data, opts), @@ -3277,7 +3238,7 @@ def _salt_cloud_force_ascii(exc): errors. """ if not isinstance(exc, (UnicodeEncodeError, UnicodeTranslateError)): - raise TypeError("Can't handle {}".format(exc)) + raise TypeError(f"Can't handle {exc}") unicode_trans = { # Convert non-breaking space to space @@ -3337,7 +3298,7 @@ def store_password_in_keyring(credential_id, username, password=None): # pylint: enable=import-error if password is None: - prompt = "Please enter password for {}: ".format(credential_id) + prompt = f"Please enter password for {credential_id}: " try: password = getpass.getpass(prompt) except EOFError: diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index a6a8a279605..d90957a0087 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -2,13 +2,12 @@ Jinja loading utils to enable a more powerful backend for jinja templates """ - import itertools import logging import os.path -import pipes import pprint import re +import shlex import time import uuid import warnings @@ -242,11 +241,11 @@ class PrintableDict(OrderedDict): if isinstance(value, str): # keeps quotes around strings # pylint: disable=repr-flag-used-in-string - output.append("{!r}: {!r}".format(key, value)) + output.append(f"{key!r}: {value!r}") # pylint: enable=repr-flag-used-in-string else: # let default output - output.append("{!r}: {!s}".format(key, value)) + output.append(f"{key!r}: {value!s}") return "{" + ", ".join(output) + "}" def __repr__(self): # pylint: disable=W0221 @@ -255,7 +254,7 @@ class PrintableDict(OrderedDict): # Raw string formatter required here because this is a repr # function. # pylint: disable=repr-flag-used-in-string - output.append("{!r}: {!r}".format(key, value)) + output.append(f"{key!r}: {value!r}") # pylint: enable=repr-flag-used-in-string return "{" + ", ".join(output) + "}" @@ -441,7 +440,7 @@ def quote(txt): 'my_text' """ - return pipes.quote(txt) + return shlex.quote(txt) @jinja_filter() @@ -1095,13 +1094,13 @@ class SerializerExtension(Extension): # to the stringified version of the exception. msg += str(exc) else: - msg += "{}\n".format(problem) + msg += f"{problem}\n" msg += salt.utils.stringutils.get_context( buf, line, marker=" <======================" ) raise TemplateRuntimeError(msg) except AttributeError: - raise TemplateRuntimeError("Unable to load yaml from {}".format(value)) + raise TemplateRuntimeError(f"Unable to load yaml from {value}") def load_json(self, value): if isinstance(value, TemplateModule): @@ -1109,7 +1108,7 @@ class SerializerExtension(Extension): try: return salt.utils.json.loads(value) except (ValueError, TypeError, AttributeError): - raise TemplateRuntimeError("Unable to load json from {}".format(value)) + raise TemplateRuntimeError(f"Unable to load json from {value}") def load_text(self, value): if isinstance(value, TemplateModule): @@ -1144,7 +1143,7 @@ class SerializerExtension(Extension): return self._parse_profile_block(parser, label, "profile block", body, lineno) def _create_profile_id(self, parser): - return "_salt_profile_{}".format(parser.free_identifier().name) + return f"_salt_profile_{parser.free_identifier().name}" def _profile_start(self, label, source): return (label, source, time.time()) @@ -1186,7 +1185,7 @@ class SerializerExtension(Extension): filter_name = parser.stream.current.value lineno = next(parser.stream).lineno if filter_name not in self.environment.filters: - parser.fail("Unable to parse {}".format(filter_name), lineno) + parser.fail(f"Unable to parse {filter_name}", lineno) parser.stream.expect("name:as") target = parser.parse_assign_target() @@ -1225,7 +1224,7 @@ class SerializerExtension(Extension): nodes.Name(target, "store").set_lineno(lineno), nodes.Filter( nodes.Name(target, "load").set_lineno(lineno), - "load_{}".format(converter), + f"load_{converter}", [], [], None, @@ -1234,7 +1233,7 @@ class SerializerExtension(Extension): ).set_lineno(lineno), ] return self._parse_profile_block( - parser, import_node.template, "import_{}".format(converter), body, lineno + parser, import_node.template, f"import_{converter}", body, lineno ) def dict_to_sls_yaml_params(self, value, flow_style=False): From 7b9957209595f26df25cadd5a74b72c7b4acd243 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 22 Feb 2023 16:57:55 +0000 Subject: [PATCH 45/53] Stop using the deprecated `imp` module Signed-off-by: Pedro Algarvio --- tests/unit/test_loader.py | 51 +++++++++++++++-------------------- tests/unit/utils/test_path.py | 11 ++++---- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/tests/unit/test_loader.py b/tests/unit/test_loader.py index cf339033200..067130620b3 100644 --- a/tests/unit/test_loader.py +++ b/tests/unit/test_loader.py @@ -8,7 +8,6 @@ import collections import compileall import copy -import imp import inspect import logging import os @@ -35,15 +34,15 @@ log = logging.getLogger(__name__) def remove_bytecode(module_path): paths = [module_path + "c"] - if hasattr(imp, "get_tag"): - modname, ext = os.path.splitext(module_path.split(os.sep)[-1]) - paths.append( - os.path.join( - os.path.dirname(module_path), - "__pycache__", - "{}.{}.pyc".format(modname, imp.get_tag()), - ) + cache_tag = sys.implementation.cache_tag + modname, ext = os.path.splitext(module_path.split(os.sep)[-1]) + paths.append( + os.path.join( + os.path.dirname(module_path), + "__pycache__", + f"{modname}.{cache_tag}.pyc", ) + ) for path in paths: if os.path.exists(path): os.unlink(path) @@ -84,9 +83,7 @@ class LazyLoaderTest(TestCase): # Setup the module self.module_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) self.addCleanup(shutil.rmtree, self.module_dir, ignore_errors=True) - self.module_file = os.path.join( - self.module_dir, "{}.py".format(self.module_name) - ) + self.module_file = os.path.join(self.module_dir, f"{self.module_name}.py") with salt.utils.files.fopen(self.module_file, "w") as fh: fh.write(salt.utils.stringutils.to_str(loader_template)) fh.flush() @@ -163,16 +160,14 @@ class LazyLoaderUtilsTest(TestCase): def setUp(self): # Setup the module self.module_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) - self.module_file = os.path.join( - self.module_dir, "{}.py".format(self.module_name) - ) + self.module_file = os.path.join(self.module_dir, f"{self.module_name}.py") with salt.utils.files.fopen(self.module_file, "w") as fh: fh.write(salt.utils.stringutils.to_str(loader_template_module)) fh.flush() os.fsync(fh.fileno()) self.utils_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) - self.utils_file = os.path.join(self.utils_dir, "{}.py".format(self.utils_name)) + self.utils_file = os.path.join(self.utils_dir, f"{self.utils_name}.py") with salt.utils.files.fopen(self.utils_file, "w") as fh: fh.write(salt.utils.stringutils.to_str(loader_template_utils)) fh.flush() @@ -516,7 +511,7 @@ class LazyLoaderSingleItem(TestCase): Checks that a KeyError is raised when the function key does not contain a '.' """ key = "testing_no_dot" - expected = "The key '{}' should contain a '.'".format(key) + expected = f"The key '{key}' should contain a '.'" with self.assertRaises(KeyError) as err: inspect.isfunction(self.loader["testing_no_dot"]) @@ -619,7 +614,7 @@ class LazyLoaderReloadingTest(TestCase): @property def module_path(self): - return os.path.join(self.tmp_dir, "{}.py".format(self.module_name)) + return os.path.join(self.tmp_dir, f"{self.module_name}.py") @pytest.mark.slow_test def test_alias(self): @@ -630,17 +625,15 @@ class LazyLoaderReloadingTest(TestCase): self.assertNotIn(self.module_key, self.loader) self.update_module() - self.assertNotIn("{}.test_alias".format(self.module_name), self.loader) + self.assertNotIn(f"{self.module_name}.test_alias", self.loader) self.assertTrue( isinstance( - self.loader["{}.working_alias".format(self.module_name)], + self.loader[f"{self.module_name}.working_alias"], salt.loader.lazy.LoadedFunc, ) ) self.assertTrue( - inspect.isfunction( - self.loader["{}.working_alias".format(self.module_name)].func - ) + inspect.isfunction(self.loader[f"{self.module_name}.working_alias"].func) ) @pytest.mark.slow_test @@ -802,7 +795,7 @@ class LazyLoaderVirtualAliasTest(TestCase): @property def module_path(self): - return os.path.join(self.tmp_dir, "{}.py".format(self.module_name)) + return os.path.join(self.tmp_dir, f"{self.module_name}.py") @pytest.mark.slow_test def test_virtual_alias(self): @@ -1199,7 +1192,7 @@ class LazyLoaderDeepSubmodReloadingTest(TestCase): "__salt__": self.minion_mods, }, ) - self.assertIn("{}.top".format(self.module_name), self.loader) + self.assertIn(f"{self.module_name}.top", self.loader) def tearDown(self): del self.tmp_dir @@ -1241,7 +1234,7 @@ class LazyLoaderDeepSubmodReloadingTest(TestCase): @pytest.mark.slow_test def test_basic(self): - self.assertIn("{}.top".format(self.module_name), self.loader) + self.assertIn(f"{self.module_name}.top", self.loader) def _verify_libs(self): for lib in self.libs: @@ -1549,9 +1542,7 @@ class LazyLoaderOptimizationOrderTest(TestCase): # Setup the module self.module_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) self.addCleanup(shutil.rmtree, self.module_dir, ignore_errors=True) - self.module_file = os.path.join( - self.module_dir, "{}.py".format(self.module_name) - ) + self.module_file = os.path.join(self.module_dir, f"{self.module_name}.py") def tearDown(self): try: @@ -1585,7 +1576,7 @@ class LazyLoaderOptimizationOrderTest(TestCase): return "lazyloadertest.cpython-{}{}{}.pyc".format( sys.version_info[0], sys.version_info[1], - "" if not optimize else ".opt-{}".format(optimize), + "" if not optimize else f".opt-{optimize}", ) def _write_module_file(self): diff --git a/tests/unit/utils/test_path.py b/tests/unit/utils/test_path.py index bebb9ce284a..47a108a2f4e 100644 --- a/tests/unit/utils/test_path.py +++ b/tests/unit/utils/test_path.py @@ -4,6 +4,7 @@ import platform import posixpath import sys import tempfile +import types import pytest @@ -40,14 +41,14 @@ class PathJoinTestCase(TestCase): def test_nix_paths(self): for idx, (parts, expected) in enumerate(self.NIX_PATHS): path = salt.utils.path.join(*parts) - assert "{}: {}".format(idx, path) == "{}: {}".format(idx, expected) + assert f"{idx}: {path}" == f"{idx}: {expected}" @pytest.mark.skip(reason="Skipped until properly mocked") @pytest.mark.skip_unless_on_windows def test_windows_paths(self): for idx, (parts, expected) in enumerate(self.WIN_PATHS): path = salt.utils.path.join(*parts) - assert "{}: {}".format(idx, path) == "{}: {}".format(idx, expected) + assert f"{idx}: {path}" == f"{idx}: {expected}" @pytest.mark.skip(reason="Skipped until properly mocked") @pytest.mark.skip_on_windows @@ -57,7 +58,7 @@ class PathJoinTestCase(TestCase): try: for idx, (parts, expected) in enumerate(self.WIN_PATHS): path = salt.utils.path.join(*parts) - assert "{}: {}".format(idx, path) == "{}: {}".format(idx, expected) + assert f"{idx}: {path}" == f"{idx}: {expected}" finally: self.__unpatch_path() @@ -79,14 +80,12 @@ class PathJoinTestCase(TestCase): assert actual == expected def __patch_path(self): - import imp - modules = list(self.BUILTIN_MODULES[:]) modules.pop(modules.index("posix")) modules.append("nt") code = """'''Salt unittest loaded NT module'''""" - module = imp.new_module("nt") + module = types.ModuleType("nt") exec(code, module.__dict__) sys.modules["nt"] = module From e4876d2e494d15c006410a8115c634e842c62885 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 14 Jun 2023 15:50:25 +0100 Subject: [PATCH 46/53] Don't hardcode the python version in the test. Signed-off-by: Pedro Algarvio --- pkg/tests/integration/test_salt_user.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/tests/integration/test_salt_user.py b/pkg/tests/integration/test_salt_user.py index d1c8d504fa0..74ec8bb0093 100644 --- a/pkg/tests/integration/test_salt_user.py +++ b/pkg/tests/integration/test_salt_user.py @@ -1,5 +1,6 @@ import pathlib import subprocess +import sys import psutil import pytest @@ -61,7 +62,9 @@ def test_salt_cloud_dirs(install_salt): Test the correct user is running the Salt Master """ paths = [ - "/opt/saltstack/salt/lib/python3.10/site-packages/salt/cloud/deploy", + "/opt/saltstack/salt/lib/python{}.{}/site-packages/salt/cloud/deploy".format( + *sys.version_info + ), "/etc/salt/cloud.deploy.d", ] for name in paths: From fb9f0bc54ade2cfa477e86b239015f0f43372558 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 16 Jun 2023 06:42:56 +0100 Subject: [PATCH 47/53] Don't hardcode the python version on `pkg/debian/salt-cloud.postinst` Signed-off-by: Pedro Algarvio --- pkg/debian/salt-cloud.postinst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/debian/salt-cloud.postinst b/pkg/debian/salt-cloud.postinst index 12a955b9349..a92551161da 100644 --- a/pkg/debian/salt-cloud.postinst +++ b/pkg/debian/salt-cloud.postinst @@ -1,5 +1,6 @@ case "$1" in configure) - chown -R salt:salt /etc/salt/cloud.deploy.d /opt/saltstack/salt/lib/python3.10/site-packages/salt/cloud/deploy + PY_VER=$(/opt/saltstack/salt/bin/python3 -c "import sys; sys.stdout.write('{}.{}'.format(*sys.version_info)); sys.stdout.flush;") + chown -R salt:salt /etc/salt/cloud.deploy.d /opt/saltstack/salt/lib/python${PY_VER}/site-packages/salt/cloud/deploy ;; esac From ba733b3a6396402241239ad0d9b90c1cbe497f6d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 14 Jun 2023 06:50:35 +0100 Subject: [PATCH 48/53] Don't hide output Signed-off-by: Pedro Algarvio --- pkg/macos/build_python.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/macos/build_python.sh b/pkg/macos/build_python.sh index b4ef1656bc4..be179b7b166 100755 --- a/pkg/macos/build_python.sh +++ b/pkg/macos/build_python.sh @@ -191,16 +191,16 @@ fi #------------------------------------------------------------------------------- _msg "Installing relenv" if [ -n "${RELENV_VERSION}" ]; then - pip install relenv==${RELENV_VERSION} >/dev/null 2>&1 - export RELENV_FETCH_VERSION=${RELENV_VERSION} + pip install relenv==${RELENV_VERSION} else - pip install relenv >/dev/null 2>&1 + pip install relenv fi if [ -n "$(relenv --version)" ]; then _success else _failure fi +export RELENV_FETCH_VERSION=$(relenv --version) #------------------------------------------------------------------------------- # Building Python with Relenv @@ -212,7 +212,7 @@ else # We want to suppress the output here so it looks nice # To see the output, remove the output redirection _msg "Fetching python (relenv)" - relenv fetch --python $PY_VERSION >/dev/null 2>&1 + relenv fetch --python=$PY_VERSION if [ -f "$RELENV_DIR/build/$PY_VERSION-x86_64-macos.tar.xz" ]; then _success else From 57e0156e64532e7e46245130d795a1cdfb45310b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 27 Jun 2023 13:01:35 +0100 Subject: [PATCH 49/53] Add change log files Signed-off-by: Pedro Algarvio --- changelog/64553.changed.md | 1 + changelog/64553.removed.md | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 changelog/64553.changed.md create mode 100644 changelog/64553.removed.md diff --git a/changelog/64553.changed.md b/changelog/64553.changed.md new file mode 100644 index 00000000000..568560cb532 --- /dev/null +++ b/changelog/64553.changed.md @@ -0,0 +1 @@ +Don't hardcode the python version on the Salt Package tests and on the `pkg/debian/salt-cloud.postinst` file diff --git a/changelog/64553.removed.md b/changelog/64553.removed.md new file mode 100644 index 00000000000..cd826073f04 --- /dev/null +++ b/changelog/64553.removed.md @@ -0,0 +1,6 @@ +Handle deprecation warnings: + +* Switch to `FullArgSpec` since Py 3.11 no longer has `ArgSpec`, deprecated since Py 3.0 +* Stop using the deprecated `cgi` module +* Stop using the deprecated `pipes` module +* Stop using the deprecated `imp` module From b6a9599a706439d855b66898581d46f67339fcde Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 27 Jun 2023 22:13:37 +0100 Subject: [PATCH 50/53] Just confirm that the expected error messages is in `stderr` Signed-off-by: Pedro Algarvio --- tests/pytests/integration/ssh/test_deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytests/integration/ssh/test_deploy.py b/tests/pytests/integration/ssh/test_deploy.py index 8512813f6e6..c95bb40a311 100644 --- a/tests/pytests/integration/ssh/test_deploy.py +++ b/tests/pytests/integration/ssh/test_deploy.py @@ -108,7 +108,7 @@ def test_retcode_exe_run_fail(salt_ssh_cli): ret = salt_ssh_cli.run("file.touch", "/tmp/non/ex/is/tent") assert ret.returncode == EX_AGGREGATE assert isinstance(ret.data, dict) - assert ret.data["stderr"] == "Error running 'file.touch': No such file or directory" + assert "Error running 'file.touch': No such file or directory" in ret.data["stderr"] assert ret.data["retcode"] == 1 From 58174eb8e13cba3034aa381a3fb35cba70358943 Mon Sep 17 00:00:00 2001 From: MKLeb Date: Tue, 27 Jun 2023 16:56:17 -0400 Subject: [PATCH 51/53] Hash the noxfile when calculating part of the cache key for our testing steps, they could be conditionally installing extra dependencies, etc... --- .github/workflows/test-action-macos.yml | 4 ++-- .github/workflows/test-action.yml | 4 ++-- .github/workflows/test-package-downloads-action-linux.yml | 4 ++-- .github/workflows/test-package-downloads-action-macos.yml | 4 ++-- .github/workflows/test-package-downloads-action-windows.yml | 4 ++-- .github/workflows/test-packages-action-macos.yml | 4 ++-- .github/workflows/test-packages-action.yml | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 3d1edefdf96..fdede59b807 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -108,7 +108,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ inputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }} + key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ inputs.python-version }}|${{ hashFiles('requirements/**/*.txt', 'noxfile.py') }} - name: Download Onedir Tarball as an Artifact if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' @@ -221,7 +221,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ inputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }} + key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ inputs.python-version }}|${{ hashFiles('requirements/**/*.txt', 'noxfile.py') }} # If we get a cache miss here it means the dependencies step failed to save the cache fail-on-cache-miss: true diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 49d75dcd86b..d3de4b9258e 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -111,7 +111,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }} - name: Download Onedir Tarball as an Artifact if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' @@ -242,7 +242,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }} # If we get a cache miss here it means the dependencies step failed to save the cache fail-on-cache-miss: true diff --git a/.github/workflows/test-package-downloads-action-linux.yml b/.github/workflows/test-package-downloads-action-linux.yml index 7aa5f65c04a..0481c1c0594 100644 --- a/.github/workflows/test-package-downloads-action-linux.yml +++ b/.github/workflows/test-package-downloads-action-linux.yml @@ -78,7 +78,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }} - name: Download Onedir Tarball as an Artifact uses: actions/download-artifact@v3 @@ -184,7 +184,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }} # If we get a cache miss here it means the dependencies step failed to save the cache fail-on-cache-miss: true diff --git a/.github/workflows/test-package-downloads-action-macos.yml b/.github/workflows/test-package-downloads-action-macos.yml index fdd1ce63991..e24ffbeed8e 100644 --- a/.github/workflows/test-package-downloads-action-macos.yml +++ b/.github/workflows/test-package-downloads-action-macos.yml @@ -80,7 +80,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }} - name: Download Onedir Tarball as an Artifact uses: actions/download-artifact@v3 @@ -184,7 +184,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }} # If we get a cache miss here it means the dependencies step failed to save the cache fail-on-cache-miss: true diff --git a/.github/workflows/test-package-downloads-action-windows.yml b/.github/workflows/test-package-downloads-action-windows.yml index 0e585f97e70..29ed67fe827 100644 --- a/.github/workflows/test-package-downloads-action-windows.yml +++ b/.github/workflows/test-package-downloads-action-windows.yml @@ -83,7 +83,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }} - name: Download Onedir Tarball as an Artifact uses: actions/download-artifact@v3 @@ -189,7 +189,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }} # If we get a cache miss here it means the dependencies step failed to save the cache fail-on-cache-miss: true diff --git a/.github/workflows/test-packages-action-macos.yml b/.github/workflows/test-packages-action-macos.yml index b7de16fb5ac..2ef20904a26 100644 --- a/.github/workflows/test-packages-action-macos.yml +++ b/.github/workflows/test-packages-action-macos.yml @@ -96,7 +96,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }} - name: Download Onedir Tarball as an Artifact if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' @@ -213,7 +213,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }} # If we get a cache miss here it means the dependencies step failed to save the cache fail-on-cache-miss: true diff --git a/.github/workflows/test-packages-action.yml b/.github/workflows/test-packages-action.yml index 71affc00877..cadaa2e834a 100644 --- a/.github/workflows/test-packages-action.yml +++ b/.github/workflows/test-packages-action.yml @@ -97,7 +97,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }} - name: Download Onedir Tarball as an Artifact if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' @@ -227,7 +227,7 @@ jobs: uses: actions/cache@v3 with: path: nox.${{ inputs.distro-slug }}.tar.* - key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }} # If we get a cache miss here it means the dependencies step failed to save the cache fail-on-cache-miss: true From 6c772d4d64e6acceb62bfd988355c418af4bc2d9 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 28 Jun 2023 07:41:56 +0100 Subject: [PATCH 52/53] Only try to get pull-request labels on pull-requests Signed-off-by: Pedro Algarvio --- .github/workflows/ci.yml | 1 + .github/workflows/nightly.yml | 1 + .github/workflows/scheduled.yml | 1 + .github/workflows/staging.yml | 1 + .github/workflows/templates/layout.yml.jinja | 1 + 5 files changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05284621f9f..62169d623d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,6 +148,7 @@ jobs: - name: Get Pull Request Test Labels id: get-pull-labels + if: ${{ github.event_name == 'pull_request'}} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0bde1d78e3c..21698717927 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -194,6 +194,7 @@ jobs: - name: Get Pull Request Test Labels id: get-pull-labels + if: ${{ github.event_name == 'pull_request'}} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 603f2d4c579..2dd28b76e10 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -184,6 +184,7 @@ jobs: - name: Get Pull Request Test Labels id: get-pull-labels + if: ${{ github.event_name == 'pull_request'}} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 7eab1044545..318f801d9c0 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -174,6 +174,7 @@ jobs: - name: Get Pull Request Test Labels id: get-pull-labels + if: ${{ github.event_name == 'pull_request'}} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/templates/layout.yml.jinja b/.github/workflows/templates/layout.yml.jinja index ea52a8df37a..426fae6a3d9 100644 --- a/.github/workflows/templates/layout.yml.jinja +++ b/.github/workflows/templates/layout.yml.jinja @@ -193,6 +193,7 @@ jobs: - name: Get Pull Request Test Labels id: get-pull-labels + if: ${{ github.event_name == 'pull_request'}} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | From 31475e74e867dffbc581556d56d56aa0009ead1a Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 28 Jun 2023 11:56:47 +0100 Subject: [PATCH 53/53] Run `pypugrade` against the files changed in the merge-forward Signed-off-by: Pedro Algarvio --- pkg/tests/integration/test_multi_minion.py | 30 ++++++++----------- salt/runner.py | 8 ++--- .../pytests/functional/cli/test_salt_run_.py | 2 +- tests/pytests/functional/states/test_user.py | 4 +-- 4 files changed, 19 insertions(+), 25 deletions(-) diff --git a/pkg/tests/integration/test_multi_minion.py b/pkg/tests/integration/test_multi_minion.py index 13d64f31f6e..ba69ab2b5ea 100644 --- a/pkg/tests/integration/test_multi_minion.py +++ b/pkg/tests/integration/test_multi_minion.py @@ -20,10 +20,9 @@ def mm_conf(mm_script): yield pathlib.Path(os.getenv("LocalAppData"), "Salt Project", "Salt", "conf") subprocess.run( ["powershell", str(mm_script).replace(" ", "' '"), "-d"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, check=False, - universal_newlines=True, + text=True, ) @@ -41,10 +40,9 @@ def test_install(mm_script, mm_conf): """ ret = subprocess.run( ["powershell", str(mm_script).replace(" ", "' '")], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, check=False, - universal_newlines=True, + text=True, ) assert ret.returncode == 0, ret.stderr conf_file = mm_conf / "minion" @@ -58,10 +56,9 @@ def test_install_master(mm_script, mm_conf): """ ret = subprocess.run( ["powershell", str(mm_script).replace(" ", "' '"), "-m", "spongebob"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, check=False, - universal_newlines=True, + text=True, ) assert ret.returncode == 0, ret.stderr conf_file = mm_conf / "minion" @@ -75,10 +72,9 @@ def test_install_prefix(mm_script, mm_conf): """ ret = subprocess.run( ["powershell", str(mm_script).replace(" ", "' '"), "-p", "squarepants"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, check=False, - universal_newlines=True, + text=True, ) assert ret.returncode == 0, ret.stderr conf_file = mm_conf / "minion" @@ -92,10 +88,9 @@ def test_install_log_level(mm_script, mm_conf): """ ret = subprocess.run( ["powershell", str(mm_script).replace(" ", "' '"), "-l", "debug"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, check=False, - universal_newlines=True, + text=True, ) assert ret.returncode == 0, ret.stderr conf_file = mm_conf / "minion" @@ -109,10 +104,9 @@ def test_install_start(mm_script, mm_conf): """ ret = subprocess.run( ["powershell", str(mm_script).replace(" ", "' '"), "-s"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, check=False, - universal_newlines=True, + text=True, ) assert ret.returncode == 0, ret.stderr conf_file = mm_conf / "minion" diff --git a/salt/runner.py b/salt/runner.py index 2a19636b8ed..d3501b8f919 100644 --- a/salt/runner.py +++ b/salt/runner.py @@ -203,7 +203,7 @@ class Runner(RunnerClient): arg = self.opts.get("fun", None) docs = super().get_docs(arg) for fun in sorted(docs): - display_output("{}:".format(fun), "text", self.opts) + display_output(f"{fun}:", "text", self.opts) print(docs[fun]) # TODO: move to mixin whenever we want a salt-wheel cli @@ -313,13 +313,13 @@ class Runner(RunnerClient): evt.fire_event( { "success": False, - "return": "{}".format(exc), + "return": f"{exc}", "retcode": 254, "fun": self.opts["fun"], "fun_args": fun_args, "jid": self.jid, }, - tag="salt/run/{}/ret".format(self.jid), + tag=f"salt/run/{self.jid}/ret", ) # Attempt to grab documentation if "fun" in low: @@ -330,7 +330,7 @@ class Runner(RunnerClient): # If we didn't get docs returned then # return the `not availble` message. if not ret: - ret = "{}".format(exc) + ret = f"{exc}" if not self.opts.get("quiet", False): display_output(ret, "nested", self.opts) else: diff --git a/tests/pytests/functional/cli/test_salt_run_.py b/tests/pytests/functional/cli/test_salt_run_.py index 66c28fc3aae..efc02e38da4 100644 --- a/tests/pytests/functional/cli/test_salt_run_.py +++ b/tests/pytests/functional/cli/test_salt_run_.py @@ -47,7 +47,7 @@ def test_versions_report(salt_run_cli): ret_lines = [line.strip() for line in ret_lines] for header in expected: - assert "{}:".format(header) in ret_lines + assert f"{header}:" in ret_lines ret_dict = {} expected_keys = set() diff --git a/tests/pytests/functional/states/test_user.py b/tests/pytests/functional/states/test_user.py index 96b1ec55c88..43ae8513012 100644 --- a/tests/pytests/functional/states/test_user.py +++ b/tests/pytests/functional/states/test_user.py @@ -348,7 +348,7 @@ def test_user_present_change_gid_but_keep_group( @pytest.mark.skip_unless_on_windows def test_user_present_existing(states, username): - win_profile = "C:\\User\\{}".format(username) + win_profile = f"C:\\User\\{username}" win_logonscript = "C:\\logon.vbs" win_description = "Test User Account" ret = states.user.present( @@ -360,7 +360,7 @@ def test_user_present_existing(states, username): ) assert ret.result is True - win_profile = "C:\\Users\\{}".format(username) + win_profile = f"C:\\Users\\{username}" win_description = "Temporary Account" ret = states.user.present( name=username,