saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Use cicd/env-cleanup-files.yml to define the cleanup patterns

Browse source 
Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>
This commit is contained in:
Pedro Algarvio 2023-01-16 18:32:08 +00:00 • committed by Pedro Algarvio
parent acf0d4182a
commit 21a8158a8f
8 changed files with 388 additions and 488 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/test-action-macos.yml vendored
View file

@ -132,7 +132,7 @@ jobs:
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
nox -e pre-archive-cleanup
nox -e "pre-archive-cleanup(pkg=False)"
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'

264
cicd/env-cleanup-files.yml Normal file
View file

@ -0,0 +1,264 @@
---
common:
dir_patterns: &common_dir_patterns
- "**/__pycache__"
# - "**/tests"
- "**/site-packages/cheroot/test"
- "**/site-packages/cherrypy/test"
- "**/site-packages/gitdb/test"
- "**/site-packages/psutil/tests"
- "**/site-packages/smmap/test"
- "**/site-packages/zmq/tests"
# Bundled Tornado Test Suite
- "**/salt/ext/tornado/test"
file_patterns: &common_file_patterns
- "*.pyc"
- "*.pyo"
ci:
darwin:
dir_patterns: &ci_darwin_dir_patterns
- *common_dir_patterns
file_patterns: &ci_darwin_file_patterns
- *common_file_patterns
linux:
dir_patterns: &ci_linux_dir_patterns
- *common_dir_patterns
file_patterns: &ci_linux_file_patterns
- *common_file_patterns
windows:
dir_patterns: &ci_windows_dir_patterns
- *common_dir_patterns
- "**/artifacts/salt/configs"
- "**/site-packages/adodbapi"
- "**/site-packages/isapi"
- "**/site-packages/pythonwin"
- "**/site-packages/win32/demos"
- "**/site-packages/tempora/tests"
- "**/site-packages/win32/test"
- "**/site-packages/win32com/test"
file_patterns: &ci_windows_file_patterns
- *common_file_patterns
# Help files
- "**/*.chm"
# Non Windows execution modules
- "**/site-packages/salt/modules/aacme.py*"
- "**/site-packages/salt/modules/aix.py*"
- "**/site-packages/salt/modules/alternatives.py*"
- "**/site-packages/salt/modules/apcups.py*"
- "**/site-packages/salt/modules/apf.py*"
- "**/site-packages/salt/modules/apt.py*"
- "**/site-packages/salt/modules/arista.py*"
- "**/site-packages/salt/modules/at.py*"
- "**/site-packages/salt/modules/bcache.py*"
- "**/site-packages/salt/modules/blockdev.py*"
- "**/site-packages/salt/modules/bluez.py*"
- "**/site-packages/salt/modules/bridge.py*"
- "**/site-packages/salt/modules/bsd.py*"
- "**/site-packages/salt/modules/btrfs.py*"
- "**/site-packages/salt/modules/ceph.py*"
- "**/site-packages/salt/modules/container_resource.py*"
- "**/site-packages/salt/modules/cron.py*"
- "**/site-packages/salt/modules/csf.py*"
- "**/site-packages/salt/modules/daemontools.py*"
- "**/site-packages/salt/modules/deb*.py*"
- "**/site-packages/salt/modules/devmap.py*"
- "**/site-packages/salt/modules/dpkg.py*"
- "**/site-packages/salt/modules/ebuild.py*"
- "**/site-packages/salt/modules/eix.py*"
- "**/site-packages/salt/modules/eselect.py*"
- "**/site-packages/salt/modules/ethtool.py*"
- "**/site-packages/salt/modules/extfs.py*"
- "**/site-packages/salt/modules/firewalld.py*"
- "**/site-packages/salt/modules/freebsd.py*"
- "**/site-packages/salt/modules/genesis.py*"
- "**/site-packages/salt/modules/gentoo.py*"
- "**/site-packages/salt/modules/glusterfs.py*"
- "**/site-packages/salt/modules/gnomedesktop.py*"
- "**/site-packages/salt/modules/groupadd.py*"
- "**/site-packages/salt/modules/grub_legacy.py*"
- "**/site-packages/salt/modules/guestfs.py*"
- "**/site-packages/salt/modules/htpasswd.py*"
- "**/site-packages/salt/modules/ilo.py*"
- "**/site-packages/salt/modules/img.py*"
- "**/site-packages/salt/modules/incron.py*"
- "**/site-packages/salt/modules/inspector.py*"
- "**/site-packages/salt/modules/ipset.py*"
- "**/site-packages/salt/modules/iptables.py*"
- "**/site-packages/salt/modules/iwtools.py*"
- "**/site-packages/salt/modules/k8s.py*"
- "**/site-packages/salt/modules/kapacitor.py*"
- "**/site-packages/salt/modules/keyboard.py*"
- "**/site-packages/salt/modules/keystone.py*"
- "**/site-packages/salt/modules/kmod.py*"
- "**/site-packages/salt/modules/layman.py*"
- "**/site-packages/salt/modules/linux.py*"
- "**/site-packages/salt/modules/localemod.py*"
- "**/site-packages/salt/modules/locate.py*"
- "**/site-packages/salt/modules/logadm.py*"
- "**/site-packages/salt/modules/logrotate.py*"
- "**/site-packages/salt/modules/lvs.py*"
- "**/site-packages/salt/modules/lxc.py*"
- "**/site-packages/salt/modules/mac.py*"
- "**/site-packages/salt/modules/makeconf.py*"
- "**/site-packages/salt/modules/mdadm.py*"
- "**/site-packages/salt/modules/mdata.py*"
- "**/site-packages/salt/modules/monit.py*"
- "**/site-packages/salt/modules/moosefs.py*"
- "**/site-packages/salt/modules/mount.py*"
- "**/site-packages/salt/modules/napalm.py*"
- "**/site-packages/salt/modules/netbsd.py*"
- "**/site-packages/salt/modules/netscaler.py*"
- "**/site-packages/salt/modules/neutron.py*"
- "**/site-packages/salt/modules/nfs3.py*"
- "**/site-packages/salt/modules/nftables.py*"
- "**/site-packages/salt/modules/nova.py*"
- "**/site-packages/salt/modules/nspawn.py*"
- "**/site-packages/salt/modules/openbsd.py*"
- "**/site-packages/salt/modules/openstack.py*"
- "**/site-packages/salt/modules/openvswitch.py*"
- "**/site-packages/salt/modules/opkg.py*"
- "**/site-packages/salt/modules/pacman.py*"
- "**/site-packages/salt/modules/parallels.py*"
- "**/site-packages/salt/modules/parted.py*"
- "**/site-packages/salt/modules/pcs.py*"
- "**/site-packages/salt/modules/pkgin.py*"
- "**/site-packages/salt/modules/pkgng.py*"
- "**/site-packages/salt/modules/pkgutil.py*"
- "**/site-packages/salt/modules/portage_config.py*"
- "**/site-packages/salt/modules/postfix.py*"
- "**/site-packages/salt/modules/poudriere.py*"
- "**/site-packages/salt/modules/powerpath.py*"
- "**/site-packages/salt/modules/pw_.py*"
- "**/site-packages/salt/modules/qemu_.py*"
- "**/site-packages/salt/modules/quota.py*"
- "**/site-packages/salt/modules/redismod.py*"
- "**/site-packages/salt/modules/restartcheck.py*"
- "**/site-packages/salt/modules/rh_.py*"
- "**/site-packages/salt/modules/riak.py*"
- "**/site-packages/salt/modules/rpm.py*"
- "**/site-packages/salt/modules/runit.py*"
- "**/site-packages/salt/modules/s6.py*"
- "**/site-packages/salt/modules/scsi.py*"
- "**/site-packages/salt/modules/seed.py*"
- "**/site-packages/salt/modules/sensors.py*"
- "**/site-packages/salt/modules/service.py*"
- "**/site-packages/salt/modules/shadow.py*"
- "**/site-packages/salt/modules/smartos.py*"
- "**/site-packages/salt/modules/smf.py*"
- "**/site-packages/salt/modules/snapper.py*"
- "**/site-packages/salt/modules/solaris.py*"
- "**/site-packages/salt/modules/solr.py*"
- "**/site-packages/salt/modules/ssh_.py*"
- "**/site-packages/salt/modules/supervisord.py*"
- "**/site-packages/salt/modules/sysbench.py*"
- "**/site-packages/salt/modules/sysfs.py*"
- "**/site-packages/salt/modules/sysrc.py*"
- "**/site-packages/salt/modules/system.py*"
- "**/site-packages/salt/modules/test_virtual.py*"
- "**/site-packages/salt/modules/timezone.py*"
- "**/site-packages/salt/modules/trafficserver.py*"
- "**/site-packages/salt/modules/tuned.py*"
- "**/site-packages/salt/modules/udev.py*"
- "**/site-packages/salt/modules/upstart.py*"
- "**/site-packages/salt/modules/useradd.py*"
- "**/site-packages/salt/modules/uswgi.py*"
- "**/site-packages/salt/modules/varnish.py*"
- "**/site-packages/salt/modules/vbox.py*"
- "**/site-packages/salt/modules/virt.py*"
- "**/site-packages/salt/modules/xapi.py*"
- "**/site-packages/salt/modules/xbpspkg.py*"
- "**/site-packages/salt/modules/xfs.py*"
- "**/site-packages/salt/modules/yum*.py*"
- "**/site-packages/salt/modules/zfs.py*"
- "**/site-packages/salt/modules/znc.py*"
- "**/site-packages/salt/modules/zpool.py*"
- "**/site-packages/salt/modules/zypper.py*"
# Non Windows state modules
- "**/site-packages/salt/states/acme.py*"
- "**/site-packages/salt/states/alternatives.py*"
- "**/site-packages/salt/states/apt.py*"
- "**/site-packages/salt/states/at.py*"
- "**/site-packages/salt/states/blockdev.py*"
- "**/site-packages/salt/states/ceph.py*"
- "**/site-packages/salt/states/cron.py*"
- "**/site-packages/salt/states/csf.py*"
- "**/site-packages/salt/states/deb.py*"
- "**/site-packages/salt/states/eselect.py*"
- "**/site-packages/salt/states/ethtool.py*"
- "**/site-packages/salt/states/firewalld.py*"
- "**/site-packages/salt/states/glusterfs.py*"
- "**/site-packages/salt/states/gnome.py*"
- "**/site-packages/salt/states/htpasswd.py*"
- "**/site-packages/salt/states/incron.py*"
- "**/site-packages/salt/states/ipset.py*"
- "**/site-packages/salt/states/iptables.py*"
- "**/site-packages/salt/states/k8s.py*"
- "**/site-packages/salt/states/kapacitor.py*"
- "**/site-packages/salt/states/keyboard.py*"
- "**/site-packages/salt/states/keystone.py*"
- "**/site-packages/salt/states/kmod.py*"
- "**/site-packages/salt/states/layman.py*"
- "**/site-packages/salt/states/linux.py*"
- "**/site-packages/salt/states/lxc.py*"
- "**/site-packages/salt/states/mac.py*"
- "**/site-packages/salt/states/makeconf.py*"
- "**/site-packages/salt/states/mdadm.py*"
- "**/site-packages/salt/states/monit.py*"
- "**/site-packages/salt/states/mount.py*"
- "**/site-packages/salt/states/nftables.py*"
- "**/site-packages/salt/states/pcs.py*"
- "**/site-packages/salt/states/pkgng.py*"
- "**/site-packages/salt/states/portage.py*"
- "**/site-packages/salt/states/powerpath.py*"
- "**/site-packages/salt/states/quota.py*"
- "**/site-packages/salt/states/redismod.py*"
- "**/site-packages/salt/states/smartos.py*"
- "**/site-packages/salt/states/snapper.py*"
- "**/site-packages/salt/states/ssh.py*"
- "**/site-packages/salt/states/supervisord.py*"
- "**/site-packages/salt/states/sysrc.py*"
- "**/site-packages/salt/states/trafficserver.py*"
- "**/site-packages/salt/states/tuned.py*"
- "**/site-packages/salt/states/vbox.py*"
- "**/site-packages/salt/states/virt.py.py*"
- "**/site-packages/salt/states/zfs.py*"
- "**/site-packages/salt/states/zpool.py*"
pkg:
darwin:
dir_patterns:
- *ci_darwin_dir_patterns
- "**/pkgconfig"
- "**/share"
- "**/artifacts/salt/opt"
- "**/artifacts/salt/etc"
- "**/artifacts/salt/Lib"
file_patterns:
- *ci_darwin_file_patterns
linux:
dir_patterns:
- *ci_linux_dir_patterns
file_patterns:
- *ci_linux_file_patterns
windows:
dir_patterns:
- *ci_windows_dir_patterns
- "**/salt/share"
file_patterns:
- *ci_windows_file_patterns
- "**/Scripts/py.exe"
- "**/Scripts/pyw.exe"
- "**/Scripts/pythonw.exe"
- "**/Scripts/venvlauncher.exe"
- "**/Scripts/venvwlauncher.exe"
- "**/Scripts/wheel*"
- "**/doc"
- "**/readme"
- "**/salt/salt-api*"
- "**/salt/salt-key*"
- "**/salt/salt-run*"
- "**/salt/salt-syndic*"
- "**/salt/salt-unity*"
- "**/salt/spm*"
- "**/salt/wheel*"

305
noxfile.py
View file

@ -1180,7 +1180,8 @@ def compress_dependencies(session):
python="3",
name="pre-archive-cleanup",
)
def pre_archive_cleanup(session):
@nox.parametrize("pkg", [False, True])
def pre_archive_cleanup(session, pkg):
"""
Call `tools pkg pre-archive-cleanup <path>`
"""
@ -1195,7 +1196,15 @@ def pre_archive_cleanup(session):
install_command = ["--progress-bar=off", "-r", requirements_file]
session.install(*install_command, silent=PIP_INSTALL_SILENT)
session_run_always(session, "tools", "pkg", "pre-archive-cleanup", ".nox")
cmdline = [
"tools",
"pkg",
"pre-archive-cleanup",
]
if pkg:
cmdline.append("--pkg")
cmdline.append(".nox")
session_run_always(session, *cmdline)
return
# On windows, we still run Py3.9
@ -1205,261 +1214,71 @@ def pre_archive_cleanup(session):
import fnmatch
import shutil
dir_patterns = [
"**/__pycache__",
# "**/tests",
"**/site-packages/cheroot/test",
"**/site-packages/cherrypy/test",
"**/site-packages/gitdb/test",
"**/site-packages/psutil/tests",
"**/site-packages/smmap/test",
"**/site-packages/zmq/tests",
]
file_patterns = [
"*.pyc",
"*.pyo",
# "test_*.py",
]
if sys.platform.lower().startswith("win"):
dir_patterns.extend(
[
"**/artifacts/salt/configs",
"**/site-packages/adodbapi",
"**/site-packages/isapi",
"**/site-packages/pythonwin",
"**/site-packages/win32/demos",
"**/site-packages/tempora/tests",
"**/site-packages/win32/test",
"**/site-packages/win32com/test",
]
)
file_patterns.extend(
[
"**/*.chm",
# "**/Scripts/py.exe",
# "**/Scripts/pyw.exe",
# "**/Scripts/pythonw.exe",
"**/Scripts/venvlauncher.exe",
"**/Scripts/venvwlauncher.exe",
# Non Windows execution modules
"**/site-packages/salt/modules/aacme.py*",
"**/site-packages/salt/modules/aix.py*",
"**/site-packages/salt/modules/alternatives.py*",
"**/site-packages/salt/modules/apcups.py*",
"**/site-packages/salt/modules/apf.py*",
"**/site-packages/salt/modules/apt.py*",
"**/site-packages/salt/modules/arista.py*",
"**/site-packages/salt/modules/at.py*",
"**/site-packages/salt/modules/bcache.py*",
"**/site-packages/salt/modules/blockdev.py*",
"**/site-packages/salt/modules/bluez.py*",
"**/site-packages/salt/modules/bridge.py*",
"**/site-packages/salt/modules/bsd.py*",
"**/site-packages/salt/modules/btrfs.py*",
"**/site-packages/salt/modules/ceph.py*",
"**/site-packages/salt/modules/container_resource.py*",
"**/site-packages/salt/modules/cron.py*",
"**/site-packages/salt/modules/csf.py*",
"**/site-packages/salt/modules/daemontools.py*",
"**/site-packages/salt/modules/deb*.py*",
"**/site-packages/salt/modules/devmap.py*",
"**/site-packages/salt/modules/dpkg.py*",
"**/site-packages/salt/modules/ebuild.py*",
"**/site-packages/salt/modules/eix.py*",
"**/site-packages/salt/modules/eselect.py*",
"**/site-packages/salt/modules/ethtool.py*",
"**/site-packages/salt/modules/extfs.py*",
"**/site-packages/salt/modules/firewalld.py*",
"**/site-packages/salt/modules/freebsd.py*",
"**/site-packages/salt/modules/genesis.py*",
"**/site-packages/salt/modules/gentoo.py*",
"**/site-packages/salt/modules/glusterfs.py*",
"**/site-packages/salt/modules/gnomedesktop.py*",
"**/site-packages/salt/modules/groupadd.py*",
"**/site-packages/salt/modules/grub_legacy.py*",
"**/site-packages/salt/modules/guestfs.py*",
"**/site-packages/salt/modules/htpasswd.py*",
"**/site-packages/salt/modules/ilo.py*",
"**/site-packages/salt/modules/img.py*",
"**/site-packages/salt/modules/incron.py*",
"**/site-packages/salt/modules/inspector.py*",
"**/site-packages/salt/modules/ipset.py*",
"**/site-packages/salt/modules/iptables.py*",
"**/site-packages/salt/modules/iwtools.py*",
"**/site-packages/salt/modules/k8s.py*",
"**/site-packages/salt/modules/kapacitor.py*",
"**/site-packages/salt/modules/keyboard.py*",
"**/site-packages/salt/modules/keystone.py*",
"**/site-packages/salt/modules/kmod.py*",
"**/site-packages/salt/modules/layman.py*",
"**/site-packages/salt/modules/linux.py*",
"**/site-packages/salt/modules/localemod.py*",
"**/site-packages/salt/modules/locate.py*",
"**/site-packages/salt/modules/logadm.py*",
"**/site-packages/salt/modules/logrotate.py*",
"**/site-packages/salt/modules/lvs.py*",
"**/site-packages/salt/modules/lxc.py*",
"**/site-packages/salt/modules/mac.py*",
"**/site-packages/salt/modules/makeconf.py*",
"**/site-packages/salt/modules/mdadm.py*",
"**/site-packages/salt/modules/mdata.py*",
"**/site-packages/salt/modules/monit.py*",
"**/site-packages/salt/modules/moosefs.py*",
"**/site-packages/salt/modules/mount.py*",
"**/site-packages/salt/modules/napalm.py*",
"**/site-packages/salt/modules/netbsd.py*",
"**/site-packages/salt/modules/netscaler.py*",
"**/site-packages/salt/modules/neutron.py*",
"**/site-packages/salt/modules/nfs3.py*",
"**/site-packages/salt/modules/nftables.py*",
"**/site-packages/salt/modules/nova.py*",
"**/site-packages/salt/modules/nspawn.py*",
"**/site-packages/salt/modules/openbsd.py*",
"**/site-packages/salt/modules/openstack.py*",
"**/site-packages/salt/modules/openvswitch.py*",
"**/site-packages/salt/modules/opkg.py*",
"**/site-packages/salt/modules/pacman.py*",
"**/site-packages/salt/modules/parallels.py*",
"**/site-packages/salt/modules/parted.py*",
"**/site-packages/salt/modules/pcs.py*",
"**/site-packages/salt/modules/pkgin.py*",
"**/site-packages/salt/modules/pkgng.py*",
"**/site-packages/salt/modules/pkgutil.py*",
"**/site-packages/salt/modules/portage_config.py*",
"**/site-packages/salt/modules/postfix.py*",
"**/site-packages/salt/modules/poudriere.py*",
"**/site-packages/salt/modules/powerpath.py*",
"**/site-packages/salt/modules/pw_.py*",
"**/site-packages/salt/modules/qemu_.py*",
"**/site-packages/salt/modules/quota.py*",
"**/site-packages/salt/modules/redismod.py*",
"**/site-packages/salt/modules/restartcheck.py*",
"**/site-packages/salt/modules/rh_.py*",
"**/site-packages/salt/modules/riak.py*",
"**/site-packages/salt/modules/rpm.py*",
"**/site-packages/salt/modules/runit.py*",
"**/site-packages/salt/modules/s6.py*",
"**/site-packages/salt/modules/scsi.py*",
"**/site-packages/salt/modules/seed.py*",
"**/site-packages/salt/modules/sensors.py*",
"**/site-packages/salt/modules/service.py*",
"**/site-packages/salt/modules/shadow.py*",
"**/site-packages/salt/modules/smartos.py*",
"**/site-packages/salt/modules/smf.py*",
"**/site-packages/salt/modules/snapper.py*",
"**/site-packages/salt/modules/solaris.py*",
"**/site-packages/salt/modules/solr.py*",
"**/site-packages/salt/modules/ssh_.py*",
"**/site-packages/salt/modules/supervisord.py*",
"**/site-packages/salt/modules/sysbench.py*",
"**/site-packages/salt/modules/sysfs.py*",
"**/site-packages/salt/modules/sysrc.py*",
"**/site-packages/salt/modules/system.py*",
"**/site-packages/salt/modules/test_virtual.py*",
"**/site-packages/salt/modules/timezone.py*",
"**/site-packages/salt/modules/trafficserver.py*",
"**/site-packages/salt/modules/tuned.py*",
"**/site-packages/salt/modules/udev.py*",
"**/site-packages/salt/modules/upstart.py*",
"**/site-packages/salt/modules/useradd.py*",
"**/site-packages/salt/modules/uswgi.py*",
"**/site-packages/salt/modules/varnish.py*",
"**/site-packages/salt/modules/vbox.py*",
"**/site-packages/salt/modules/virt.py*",
"**/site-packages/salt/modules/xapi.py*",
"**/site-packages/salt/modules/xbpspkg.py*",
"**/site-packages/salt/modules/xfs.py*",
"**/site-packages/salt/modules/yum*.py*",
"**/site-packages/salt/modules/zfs.py*",
"**/site-packages/salt/modules/znc.py*",
"**/site-packages/salt/modules/zpool.py*",
"**/site-packages/salt/modules/zypper.py*",
# Non Windows state modules
"**/site-packages/salt/states/acme.py*",
"**/site-packages/salt/states/alternatives.py*",
"**/site-packages/salt/states/apt.py*",
"**/site-packages/salt/states/at.py*",
"**/site-packages/salt/states/blockdev.py*",
"**/site-packages/salt/states/ceph.py*",
"**/site-packages/salt/states/cron.py*",
"**/site-packages/salt/states/csf.py*",
"**/site-packages/salt/states/deb.py*",
"**/site-packages/salt/states/eselect.py*",
"**/site-packages/salt/states/ethtool.py*",
"**/site-packages/salt/states/firewalld.py*",
"**/site-packages/salt/states/glusterfs.py*",
"**/site-packages/salt/states/gnome.py*",
"**/site-packages/salt/states/htpasswd.py*",
"**/site-packages/salt/states/incron.py*",
"**/site-packages/salt/states/ipset.py*",
"**/site-packages/salt/states/iptables.py*",
"**/site-packages/salt/states/k8s.py*",
"**/site-packages/salt/states/kapacitor.py*",
"**/site-packages/salt/states/keyboard.py*",
"**/site-packages/salt/states/keystone.py*",
"**/site-packages/salt/states/kmod.py*",
"**/site-packages/salt/states/layman.py*",
"**/site-packages/salt/states/linux.py*",
"**/site-packages/salt/states/lxc.py*",
"**/site-packages/salt/states/mac.py*",
"**/site-packages/salt/states/makeconf.py*",
"**/site-packages/salt/states/mdadm.py*",
"**/site-packages/salt/states/monit.py*",
"**/site-packages/salt/states/mount.py*",
"**/site-packages/salt/states/nftables.py*",
"**/site-packages/salt/states/pcs.py*",
"**/site-packages/salt/states/pkgng.py*",
"**/site-packages/salt/states/portage.py*",
"**/site-packages/salt/states/powerpath.py*",
"**/site-packages/salt/states/quota.py*",
"**/site-packages/salt/states/redismod.py*",
"**/site-packages/salt/states/smartos.py*",
"**/site-packages/salt/states/snapper.py*",
"**/site-packages/salt/states/ssh.py*",
"**/site-packages/salt/states/supervisord.py*",
"**/site-packages/salt/states/sysrc.py*",
"**/site-packages/salt/states/trafficserver.py*",
"**/site-packages/salt/states/tuned.py*",
"**/site-packages/salt/states/vbox.py*",
"**/site-packages/salt/states/virt.py.py*",
"**/site-packages/salt/states/zfs.py*",
"**/site-packages/salt/states/zpool.py*",
]
)
if sys.platform.lower().startswith("darwin"):
dir_patterns.extend(
[
"**/pkgconfig",
"**/share",
"**/artifacts/salt/opt",
"**/artifacts/salt/etc",
"**/artifacts/salt/Lib",
]
)
try:
import yaml
except ImportError:
session.error("Please install 'pyyaml'.")
return
with open(str(REPO_ROOT / "cicd" / "env-cleanup-files.yml")) as rfh:
patterns = yaml.safe_load(rfh.read())
if pkg:
patterns = patterns["pkg"]
else:
patterns = patterns["ci"]
if IS_WINDOWS:
patterns = patterns["windows"]
elif IS_DARWIN:
patterns = patterns["darwin"]
else:
patterns = patterns["linux"]
dir_patterns = set()
for pattern in patterns["dir_patterns"]:
if isinstance(pattern, list):
dir_patterns.update(set(pattern))
continue
dir_patterns.add(pattern)
file_patterns = set()
for pattern in patterns["file_patterns"]:
if isinstance(pattern, list):
file_patterns.update(set(pattern))
continue
file_patterns.add(pattern)
for root, dirs, files in os.walk(
str(REPO_ROOT / ".nox"), topdown=True, followlinks=False
):
for dirname in dirs:
path = os.path.join(root, dirname)
if not os.path.exists(path):
path = pathlib.Path(root, dirname).resolve()
if not path.exists():
continue
match_path = pathlib.Path(path).as_posix()
match_path = path.as_posix()
for pattern in dir_patterns:
if fnmatch.fnmatch(str(match_path), pattern):
session.log(f"Deleting directory: {match_path}")
shutil.rmtree(path)
session.log(
f"Deleting directory: {match_path}; Matching pattern: {pattern!r}"
)
shutil.rmtree(str(path))
break
for filename in files:
path = os.path.join(root, filename)
if not os.path.exists(path):
path = pathlib.Path(root, filename).resolve()
if not path.exists():
continue
match_path = pathlib.Path(path).as_posix()
match_path = path.as_posix()
for pattern in file_patterns:
if fnmatch.fnmatch(str(match_path), pattern):
session.log(f"Deleting file: {match_path}")
session.log(
f"Deleting file: {match_path}; Matching pattern: {pattern!r}"
)
try:
os.remove(path)
os.remove(str(path))
except FileNotFoundError:
pass
break
@nox.session(python="3", name="combine-coverage")

2
requirements/static/ci/py3.10/tools.txt
View file

@ -26,6 +26,8 @@ python-dateutil==2.8.2
# via botocore
python-tools-scripts==0.9.3
# via -r requirements/static/ci/tools.in
pyyaml==6.0
# via -r requirements/static/ci/tools.in
rich==12.5.1
# via python-tools-scripts
s3transfer==0.5.2

2
requirements/static/ci/py3.9/tools.txt
View file

@ -26,6 +26,8 @@ python-dateutil==2.8.2
# via botocore
python-tools-scripts==0.9.3
# via -r requirements/static/ci/tools.in
pyyaml==6.0
# via -r requirements/static/ci/tools.in
rich==12.5.1
# via python-tools-scripts
s3transfer==0.5.2

1
requirements/static/ci/tools.in
View file

@ -1,3 +1,4 @@
python-tools-scripts >= 0.9.3
attrs
boto3
pyyaml

293
tools/pkg.py
View file

@ -11,6 +11,7 @@ import pathlib
import shutil
import sys
import yaml
from ptscripts import Context, command_group
log = logging.getLogger(__name__)
@ -93,16 +94,19 @@ def set_salt_version(ctx: Context, salt_version: str, overwrite: bool = False):
@pkg.command(
name="pre-archive-cleanup",
arguments={
"path": {
"cleanup_path": {
"help": (
"The salt version to write to 'salt/_version.txt'. If not passed "
"it will be discovered by running 'python3 salt/version.py'."
),
"metavar": "PATH_TO_CLEANUP",
},
"pkg": {
"help": "Perform extended, pre-packaging cleanup routines",
},
},
)
def pre_archive_cleanup(ctx: Context, path: str):
def pre_archive_cleanup(ctx: Context, cleanup_path: str, pkg: bool = False):
"""
Clean the provided path of paths that shouyld not be included in the archive.
@ -114,255 +118,60 @@ def pre_archive_cleanup(ctx: Context, path: str):
When running on Windows and macOS, some additional cleanup is also done.
"""
dir_patterns = [
"**/__pycache__",
# "**/tests",
"**/site-packages/cheroot/test",
"**/site-packages/cherrypy/test",
"**/site-packages/gitdb/test",
"**/site-packages/psutil/tests",
"**/site-packages/smmap/test",
"**/site-packages/zmq/tests",
]
file_patterns = [
"*.pyc",
"*.pyo",
# "test_*.py",
]
with open(str(REPO_ROOT / "cicd" / "env-cleanup-files.yml")) as rfh:
patterns = yaml.safe_load(rfh.read())
if pkg:
patterns = patterns["pkg"]
else:
patterns = patterns["ci"]
if sys.platform.lower().startswith("win"):
dir_patterns.extend(
[
"**/artifacts/salt/configs",
"**/site-packages/adodbapi",
"**/site-packages/isapi",
"**/site-packages/pythonwin",
"**/site-packages/win32/demos",
"**/site-packages/tempora/tests",
"**/site-packages/win32/test",
"**/site-packages/win32com/test",
]
)
file_patterns.extend(
[
"**/*.chm" "**/Scripts/py.exe",
"**/Scripts/pyw.exe",
"**/Scripts/pythonw.exe",
"**/Scripts/venvlauncher.exe",
"**/Scripts/venvwlauncher.exe",
# Non Windows execution modules
"**/site-packages/salt/modules/aacme.py*",
"**/site-packages/salt/modules/aix.py*",
"**/site-packages/salt/modules/alternatives.py*",
"**/site-packages/salt/modules/apcups.py*",
"**/site-packages/salt/modules/apf.py*",
"**/site-packages/salt/modules/apt.py*",
"**/site-packages/salt/modules/arista.py*",
"**/site-packages/salt/modules/at.py*",
"**/site-packages/salt/modules/bcache.py*",
"**/site-packages/salt/modules/blockdev.py*",
"**/site-packages/salt/modules/bluez.py*",
"**/site-packages/salt/modules/bridge.py*",
"**/site-packages/salt/modules/bsd.py*",
"**/site-packages/salt/modules/btrfs.py*",
"**/site-packages/salt/modules/ceph.py*",
"**/site-packages/salt/modules/container_resource.py*",
"**/site-packages/salt/modules/cron.py*",
"**/site-packages/salt/modules/csf.py*",
"**/site-packages/salt/modules/daemontools.py*",
"**/site-packages/salt/modules/deb*.py*",
"**/site-packages/salt/modules/devmap.py*",
"**/site-packages/salt/modules/dpkg.py*",
"**/site-packages/salt/modules/ebuild.py*",
"**/site-packages/salt/modules/eix.py*",
"**/site-packages/salt/modules/eselect.py*",
"**/site-packages/salt/modules/ethtool.py*",
"**/site-packages/salt/modules/extfs.py*",
"**/site-packages/salt/modules/firewalld.py*",
"**/site-packages/salt/modules/freebsd.py*",
"**/site-packages/salt/modules/genesis.py*",
"**/site-packages/salt/modules/gentoo.py*",
"**/site-packages/salt/modules/glusterfs.py*",
"**/site-packages/salt/modules/gnomedesktop.py*",
"**/site-packages/salt/modules/groupadd.py*",
"**/site-packages/salt/modules/grub_legacy.py*",
"**/site-packages/salt/modules/guestfs.py*",
"**/site-packages/salt/modules/htpasswd.py*",
"**/site-packages/salt/modules/ilo.py*",
"**/site-packages/salt/modules/img.py*",
"**/site-packages/salt/modules/incron.py*",
"**/site-packages/salt/modules/inspector.py*",
"**/site-packages/salt/modules/ipset.py*",
"**/site-packages/salt/modules/iptables.py*",
"**/site-packages/salt/modules/iwtools.py*",
"**/site-packages/salt/modules/k8s.py*",
"**/site-packages/salt/modules/kapacitor.py*",
"**/site-packages/salt/modules/keyboard.py*",
"**/site-packages/salt/modules/keystone.py*",
"**/site-packages/salt/modules/kmod.py*",
"**/site-packages/salt/modules/layman.py*",
"**/site-packages/salt/modules/linux.py*",
"**/site-packages/salt/modules/localemod.py*",
"**/site-packages/salt/modules/locate.py*",
"**/site-packages/salt/modules/logadm.py*",
"**/site-packages/salt/modules/logrotate.py*",
"**/site-packages/salt/modules/lvs.py*",
"**/site-packages/salt/modules/lxc.py*",
"**/site-packages/salt/modules/mac.py*",
"**/site-packages/salt/modules/makeconf.py*",
"**/site-packages/salt/modules/mdadm.py*",
"**/site-packages/salt/modules/mdata.py*",
"**/site-packages/salt/modules/monit.py*",
"**/site-packages/salt/modules/moosefs.py*",
"**/site-packages/salt/modules/mount.py*",
"**/site-packages/salt/modules/napalm.py*",
"**/site-packages/salt/modules/netbsd.py*",
"**/site-packages/salt/modules/netscaler.py*",
"**/site-packages/salt/modules/neutron.py*",
"**/site-packages/salt/modules/nfs3.py*",
"**/site-packages/salt/modules/nftables.py*",
"**/site-packages/salt/modules/nova.py*",
"**/site-packages/salt/modules/nspawn.py*",
"**/site-packages/salt/modules/openbsd.py*",
"**/site-packages/salt/modules/openstack.py*",
"**/site-packages/salt/modules/openvswitch.py*",
"**/site-packages/salt/modules/opkg.py*",
"**/site-packages/salt/modules/pacman.py*",
"**/site-packages/salt/modules/parallels.py*",
"**/site-packages/salt/modules/parted.py*",
"**/site-packages/salt/modules/pcs.py*",
"**/site-packages/salt/modules/pkgin.py*",
"**/site-packages/salt/modules/pkgng.py*",
"**/site-packages/salt/modules/pkgutil.py*",
"**/site-packages/salt/modules/portage_config.py*",
"**/site-packages/salt/modules/postfix.py*",
"**/site-packages/salt/modules/poudriere.py*",
"**/site-packages/salt/modules/powerpath.py*",
"**/site-packages/salt/modules/pw_.py*",
"**/site-packages/salt/modules/qemu_.py*",
"**/site-packages/salt/modules/quota.py*",
"**/site-packages/salt/modules/redismod.py*",
"**/site-packages/salt/modules/restartcheck.py*",
"**/site-packages/salt/modules/rh_.py*",
"**/site-packages/salt/modules/riak.py*",
"**/site-packages/salt/modules/rpm.py*",
"**/site-packages/salt/modules/runit.py*",
"**/site-packages/salt/modules/s6.py*",
"**/site-packages/salt/modules/scsi.py*",
"**/site-packages/salt/modules/seed.py*",
"**/site-packages/salt/modules/sensors.py*",
"**/site-packages/salt/modules/service.py*",
"**/site-packages/salt/modules/shadow.py*",
"**/site-packages/salt/modules/smartos.py*",
"**/site-packages/salt/modules/smf.py*",
"**/site-packages/salt/modules/snapper.py*",
"**/site-packages/salt/modules/solaris.py*",
"**/site-packages/salt/modules/solr.py*",
"**/site-packages/salt/modules/ssh_.py*",
"**/site-packages/salt/modules/supervisord.py*",
"**/site-packages/salt/modules/sysbench.py*",
"**/site-packages/salt/modules/sysfs.py*",
"**/site-packages/salt/modules/sysrc.py*",
"**/site-packages/salt/modules/system.py*",
"**/site-packages/salt/modules/test_virtual.py*",
"**/site-packages/salt/modules/timezone.py*",
"**/site-packages/salt/modules/trafficserver.py*",
"**/site-packages/salt/modules/tuned.py*",
"**/site-packages/salt/modules/udev.py*",
"**/site-packages/salt/modules/upstart.py*",
"**/site-packages/salt/modules/useradd.py*",
"**/site-packages/salt/modules/uswgi.py*",
"**/site-packages/salt/modules/varnish.py*",
"**/site-packages/salt/modules/vbox.py*",
"**/site-packages/salt/modules/virt.py*",
"**/site-packages/salt/modules/xapi.py*",
"**/site-packages/salt/modules/xbpspkg.py*",
"**/site-packages/salt/modules/xfs.py*",
"**/site-packages/salt/modules/yum*.py*",
"**/site-packages/salt/modules/zfs.py*",
"**/site-packages/salt/modules/znc.py*",
"**/site-packages/salt/modules/zpool.py*",
"**/site-packages/salt/modules/zypper.py*",
# Non Windows state modules
"**/site-packages/salt/states/acme.py*",
"**/site-packages/salt/states/alternatives.py*",
"**/site-packages/salt/states/apt.py*",
"**/site-packages/salt/states/at.py*",
"**/site-packages/salt/states/blockdev.py*",
"**/site-packages/salt/states/ceph.py*",
"**/site-packages/salt/states/cron.py*",
"**/site-packages/salt/states/csf.py*",
"**/site-packages/salt/states/deb.py*",
"**/site-packages/salt/states/eselect.py*",
"**/site-packages/salt/states/ethtool.py*",
"**/site-packages/salt/states/firewalld.py*",
"**/site-packages/salt/states/glusterfs.py*",
"**/site-packages/salt/states/gnome.py*",
"**/site-packages/salt/states/htpasswd.py*",
"**/site-packages/salt/states/incron.py*",
"**/site-packages/salt/states/ipset.py*",
"**/site-packages/salt/states/iptables.py*",
"**/site-packages/salt/states/k8s.py*",
"**/site-packages/salt/states/kapacitor.py*",
"**/site-packages/salt/states/keyboard.py*",
"**/site-packages/salt/states/keystone.py*",
"**/site-packages/salt/states/kmod.py*",
"**/site-packages/salt/states/layman.py*",
"**/site-packages/salt/states/linux.py*",
"**/site-packages/salt/states/lxc.py*",
"**/site-packages/salt/states/mac.py*",
"**/site-packages/salt/states/makeconf.py*",
"**/site-packages/salt/states/mdadm.py*",
"**/site-packages/salt/states/monit.py*",
"**/site-packages/salt/states/mount.py*",
"**/site-packages/salt/states/nftables.py*",
"**/site-packages/salt/states/pcs.py*",
"**/site-packages/salt/states/pkgng.py*",
"**/site-packages/salt/states/portage.py*",
"**/site-packages/salt/states/powerpath.py*",
"**/site-packages/salt/states/quota.py*",
"**/site-packages/salt/states/redismod.py*",
"**/site-packages/salt/states/smartos.py*",
"**/site-packages/salt/states/snapper.py*",
"**/site-packages/salt/states/ssh.py*",
"**/site-packages/salt/states/supervisord.py*",
"**/site-packages/salt/states/sysrc.py*",
"**/site-packages/salt/states/trafficserver.py*",
"**/site-packages/salt/states/tuned.py*",
"**/site-packages/salt/states/vbox.py*",
"**/site-packages/salt/states/virt.py.py*",
"**/site-packages/salt/states/zfs.py*",
"**/site-packages/salt/states/zpool.py*",
]
)
if sys.platform.lower().startswith("darwin"):
dir_patterns.extend(
[
"**/pkgconfig",
"**/share",
"**/artifacts/salt/opt",
"**/artifacts/salt/etc",
"**/artifacts/salt/Lib",
]
)
for root, dirs, files in os.walk(path, topdown=True, followlinks=False):
patterns = patterns["windows"]
elif sys.platform.lower().startswith("darwin"):
patterns = patterns["darwin"]
else:
patterns = patterns["linux"]
dir_patterns = set()
for pattern in patterns["dir_patterns"]:
if isinstance(pattern, list):
dir_patterns.update(set(pattern))
continue
dir_patterns.add(pattern)
file_patterns = set()
for pattern in patterns["file_patterns"]:
if isinstance(pattern, list):
file_patterns.update(set(pattern))
continue
file_patterns.add(pattern)
for root, dirs, files in os.walk(cleanup_path, topdown=True, followlinks=False):
for dirname in dirs:
path = os.path.join(root, dirname)
if not os.path.exists(path):
path = pathlib.Path(root, dirname).resolve()
if not path.exists():
continue
match_path = pathlib.Path(path).as_posix()
match_path = path.as_posix()
for pattern in dir_patterns:
if fnmatch.fnmatch(str(match_path), pattern):
ctx.info(f"Deleting directory: {match_path}")
shutil.rmtree(path)
ctx.info(
f"Deleting directory: {match_path}; Matching pattern: {pattern!r}"
)
shutil.rmtree(str(path))
break
for filename in files:
path = os.path.join(root, filename)
if not os.path.exists(path):
path = pathlib.Path(root, filename).resolve()
if not path.exists():
continue
match_path = pathlib.Path(path).as_posix()
match_path = path.as_posix()
for pattern in file_patterns:
if fnmatch.fnmatch(str(match_path), pattern):
ctx.info(f"Deleting file: {match_path}")
ctx.info(
f"Deleting file: {match_path}; Matching pattern: {pattern!r}"
)
try:
os.remove(path)
os.remove(str(path))
except FileNotFoundError:
pass
break

7
tools/vm.py
View file

@ -370,14 +370,17 @@ def install_dependencies(ctx: Context, name: str, nox_session: str = "ci-test-3"
"help": "The VM Name",
"metavar": "VM_NAME",
},
"pkg": {
"help": "Perform extended, pre-packaging cleanup routines",
},
},
)
def pre_archive_cleanup(ctx: Context, name: str):
def pre_archive_cleanup(ctx: Context, name: str, pkg: bool = False):
"""
Pre `.nox` directory compress cleanup.
"""
vm = VM(ctx=ctx, name=name, region_name=ctx.parser.options.region)
returncode = vm.run_nox("pre-archive-cleanup")
returncode = vm.run_nox(f"pre-archive-cleanup(pkg={pkg})")
ctx.exit(returncode)