saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Cleanup .nox directory before compressing dependencies

Browse source 
Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>
This commit is contained in:
Pedro Algarvio 2023-01-15 11:16:42 +00:00 • committed by Pedro Algarvio
parent 15b9f9231b
commit acf0d4182a
4 changed files with 338 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.github/workflows/test-action-macos.yml vendored
View file

@ -129,6 +129,11 @@ jobs:
export PKG_CONFIG_PATH="/usr/local/opt/openssl@3/lib/pkgconfig"
nox --install-only -e ${{ env.NOX_SESSION }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
nox -e pre-archive-cleanup
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |

5
.github/workflows/test-action.yml vendored
View file

@ -123,6 +123,11 @@ jobs:
run: |
tools --timestamps vm install-dependencies --nox-session=${{ env.NOX_SESSION }} ${{ inputs.distro-slug }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm pre-archive-cleanup ${{ inputs.distro-slug }}
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |

295
noxfile.py
View file

@ -1166,7 +1166,300 @@ def compress_dependencies(session):
)
nox_dependencies_tarball_path.unlink()
session_run_always(session, "tar", "-caf", nox_dependencies_tarball, ".nox")
session_run_always(
session,
"tar",
"-caf",
nox_dependencies_tarball,
"--exclude=.nox/pre-archive-cleanup",
".nox",
)
@nox.session(
python="3",
name="pre-archive-cleanup",
)
def pre_archive_cleanup(session):
"""
Call `tools pkg pre-archive-cleanup <path>`
"""
if session.posargs:
session.error("No additional arguments can be passed to 'pre-archive-cleanup'")
version_info = _get_session_python_version_info(session)
if version_info >= (3, 9):
if _upgrade_pip_setuptools_and_wheel(session):
requirements_file = os.path.join(
"requirements", "static", "ci", _get_pydir(session), "tools.txt"
)
install_command = ["--progress-bar=off", "-r", requirements_file]
session.install(*install_command, silent=PIP_INSTALL_SILENT)
session_run_always(session, "tools", "pkg", "pre-archive-cleanup", ".nox")
return
# On windows, we still run Py3.9
# Let's do the cleanup here, for now.
# This is a copy of the pre_archive_cleanup function in tools/pkg.py
import fnmatch
import shutil
dir_patterns = [
"**/__pycache__",
# "**/tests",
"**/site-packages/cheroot/test",
"**/site-packages/cherrypy/test",
"**/site-packages/gitdb/test",
"**/site-packages/psutil/tests",
"**/site-packages/smmap/test",
"**/site-packages/zmq/tests",
]
file_patterns = [
"*.pyc",
"*.pyo",
# "test_*.py",
]
if sys.platform.lower().startswith("win"):
dir_patterns.extend(
[
"**/artifacts/salt/configs",
"**/site-packages/adodbapi",
"**/site-packages/isapi",
"**/site-packages/pythonwin",
"**/site-packages/win32/demos",
"**/site-packages/tempora/tests",
"**/site-packages/win32/test",
"**/site-packages/win32com/test",
]
)
file_patterns.extend(
[
"**/*.chm",
# "**/Scripts/py.exe",
# "**/Scripts/pyw.exe",
# "**/Scripts/pythonw.exe",
"**/Scripts/venvlauncher.exe",
"**/Scripts/venvwlauncher.exe",
# Non Windows execution modules
"**/site-packages/salt/modules/aacme.py*",
"**/site-packages/salt/modules/aix.py*",
"**/site-packages/salt/modules/alternatives.py*",
"**/site-packages/salt/modules/apcups.py*",
"**/site-packages/salt/modules/apf.py*",
"**/site-packages/salt/modules/apt.py*",
"**/site-packages/salt/modules/arista.py*",
"**/site-packages/salt/modules/at.py*",
"**/site-packages/salt/modules/bcache.py*",
"**/site-packages/salt/modules/blockdev.py*",
"**/site-packages/salt/modules/bluez.py*",
"**/site-packages/salt/modules/bridge.py*",
"**/site-packages/salt/modules/bsd.py*",
"**/site-packages/salt/modules/btrfs.py*",
"**/site-packages/salt/modules/ceph.py*",
"**/site-packages/salt/modules/container_resource.py*",
"**/site-packages/salt/modules/cron.py*",
"**/site-packages/salt/modules/csf.py*",
"**/site-packages/salt/modules/daemontools.py*",
"**/site-packages/salt/modules/deb*.py*",
"**/site-packages/salt/modules/devmap.py*",
"**/site-packages/salt/modules/dpkg.py*",
"**/site-packages/salt/modules/ebuild.py*",
"**/site-packages/salt/modules/eix.py*",
"**/site-packages/salt/modules/eselect.py*",
"**/site-packages/salt/modules/ethtool.py*",
"**/site-packages/salt/modules/extfs.py*",
"**/site-packages/salt/modules/firewalld.py*",
"**/site-packages/salt/modules/freebsd.py*",
"**/site-packages/salt/modules/genesis.py*",
"**/site-packages/salt/modules/gentoo.py*",
"**/site-packages/salt/modules/glusterfs.py*",
"**/site-packages/salt/modules/gnomedesktop.py*",
"**/site-packages/salt/modules/groupadd.py*",
"**/site-packages/salt/modules/grub_legacy.py*",
"**/site-packages/salt/modules/guestfs.py*",
"**/site-packages/salt/modules/htpasswd.py*",
"**/site-packages/salt/modules/ilo.py*",
"**/site-packages/salt/modules/img.py*",
"**/site-packages/salt/modules/incron.py*",
"**/site-packages/salt/modules/inspector.py*",
"**/site-packages/salt/modules/ipset.py*",
"**/site-packages/salt/modules/iptables.py*",
"**/site-packages/salt/modules/iwtools.py*",
"**/site-packages/salt/modules/k8s.py*",
"**/site-packages/salt/modules/kapacitor.py*",
"**/site-packages/salt/modules/keyboard.py*",
"**/site-packages/salt/modules/keystone.py*",
"**/site-packages/salt/modules/kmod.py*",
"**/site-packages/salt/modules/layman.py*",
"**/site-packages/salt/modules/linux.py*",
"**/site-packages/salt/modules/localemod.py*",
"**/site-packages/salt/modules/locate.py*",
"**/site-packages/salt/modules/logadm.py*",
"**/site-packages/salt/modules/logrotate.py*",
"**/site-packages/salt/modules/lvs.py*",
"**/site-packages/salt/modules/lxc.py*",
"**/site-packages/salt/modules/mac.py*",
"**/site-packages/salt/modules/makeconf.py*",
"**/site-packages/salt/modules/mdadm.py*",
"**/site-packages/salt/modules/mdata.py*",
"**/site-packages/salt/modules/monit.py*",
"**/site-packages/salt/modules/moosefs.py*",
"**/site-packages/salt/modules/mount.py*",
"**/site-packages/salt/modules/napalm.py*",
"**/site-packages/salt/modules/netbsd.py*",
"**/site-packages/salt/modules/netscaler.py*",
"**/site-packages/salt/modules/neutron.py*",
"**/site-packages/salt/modules/nfs3.py*",
"**/site-packages/salt/modules/nftables.py*",
"**/site-packages/salt/modules/nova.py*",
"**/site-packages/salt/modules/nspawn.py*",
"**/site-packages/salt/modules/openbsd.py*",
"**/site-packages/salt/modules/openstack.py*",
"**/site-packages/salt/modules/openvswitch.py*",
"**/site-packages/salt/modules/opkg.py*",
"**/site-packages/salt/modules/pacman.py*",
"**/site-packages/salt/modules/parallels.py*",
"**/site-packages/salt/modules/parted.py*",
"**/site-packages/salt/modules/pcs.py*",
"**/site-packages/salt/modules/pkgin.py*",
"**/site-packages/salt/modules/pkgng.py*",
"**/site-packages/salt/modules/pkgutil.py*",
"**/site-packages/salt/modules/portage_config.py*",
"**/site-packages/salt/modules/postfix.py*",
"**/site-packages/salt/modules/poudriere.py*",
"**/site-packages/salt/modules/powerpath.py*",
"**/site-packages/salt/modules/pw_.py*",
"**/site-packages/salt/modules/qemu_.py*",
"**/site-packages/salt/modules/quota.py*",
"**/site-packages/salt/modules/redismod.py*",
"**/site-packages/salt/modules/restartcheck.py*",
"**/site-packages/salt/modules/rh_.py*",
"**/site-packages/salt/modules/riak.py*",
"**/site-packages/salt/modules/rpm.py*",
"**/site-packages/salt/modules/runit.py*",
"**/site-packages/salt/modules/s6.py*",
"**/site-packages/salt/modules/scsi.py*",
"**/site-packages/salt/modules/seed.py*",
"**/site-packages/salt/modules/sensors.py*",
"**/site-packages/salt/modules/service.py*",
"**/site-packages/salt/modules/shadow.py*",
"**/site-packages/salt/modules/smartos.py*",
"**/site-packages/salt/modules/smf.py*",
"**/site-packages/salt/modules/snapper.py*",
"**/site-packages/salt/modules/solaris.py*",
"**/site-packages/salt/modules/solr.py*",
"**/site-packages/salt/modules/ssh_.py*",
"**/site-packages/salt/modules/supervisord.py*",
"**/site-packages/salt/modules/sysbench.py*",
"**/site-packages/salt/modules/sysfs.py*",
"**/site-packages/salt/modules/sysrc.py*",
"**/site-packages/salt/modules/system.py*",
"**/site-packages/salt/modules/test_virtual.py*",
"**/site-packages/salt/modules/timezone.py*",
"**/site-packages/salt/modules/trafficserver.py*",
"**/site-packages/salt/modules/tuned.py*",
"**/site-packages/salt/modules/udev.py*",
"**/site-packages/salt/modules/upstart.py*",
"**/site-packages/salt/modules/useradd.py*",
"**/site-packages/salt/modules/uswgi.py*",
"**/site-packages/salt/modules/varnish.py*",
"**/site-packages/salt/modules/vbox.py*",
"**/site-packages/salt/modules/virt.py*",
"**/site-packages/salt/modules/xapi.py*",
"**/site-packages/salt/modules/xbpspkg.py*",
"**/site-packages/salt/modules/xfs.py*",
"**/site-packages/salt/modules/yum*.py*",
"**/site-packages/salt/modules/zfs.py*",
"**/site-packages/salt/modules/znc.py*",
"**/site-packages/salt/modules/zpool.py*",
"**/site-packages/salt/modules/zypper.py*",
# Non Windows state modules
"**/site-packages/salt/states/acme.py*",
"**/site-packages/salt/states/alternatives.py*",
"**/site-packages/salt/states/apt.py*",
"**/site-packages/salt/states/at.py*",
"**/site-packages/salt/states/blockdev.py*",
"**/site-packages/salt/states/ceph.py*",
"**/site-packages/salt/states/cron.py*",
"**/site-packages/salt/states/csf.py*",
"**/site-packages/salt/states/deb.py*",
"**/site-packages/salt/states/eselect.py*",
"**/site-packages/salt/states/ethtool.py*",
"**/site-packages/salt/states/firewalld.py*",
"**/site-packages/salt/states/glusterfs.py*",
"**/site-packages/salt/states/gnome.py*",
"**/site-packages/salt/states/htpasswd.py*",
"**/site-packages/salt/states/incron.py*",
"**/site-packages/salt/states/ipset.py*",
"**/site-packages/salt/states/iptables.py*",
"**/site-packages/salt/states/k8s.py*",
"**/site-packages/salt/states/kapacitor.py*",
"**/site-packages/salt/states/keyboard.py*",
"**/site-packages/salt/states/keystone.py*",
"**/site-packages/salt/states/kmod.py*",
"**/site-packages/salt/states/layman.py*",
"**/site-packages/salt/states/linux.py*",
"**/site-packages/salt/states/lxc.py*",
"**/site-packages/salt/states/mac.py*",
"**/site-packages/salt/states/makeconf.py*",
"**/site-packages/salt/states/mdadm.py*",
"**/site-packages/salt/states/monit.py*",
"**/site-packages/salt/states/mount.py*",
"**/site-packages/salt/states/nftables.py*",
"**/site-packages/salt/states/pcs.py*",
"**/site-packages/salt/states/pkgng.py*",
"**/site-packages/salt/states/portage.py*",
"**/site-packages/salt/states/powerpath.py*",
"**/site-packages/salt/states/quota.py*",
"**/site-packages/salt/states/redismod.py*",
"**/site-packages/salt/states/smartos.py*",
"**/site-packages/salt/states/snapper.py*",
"**/site-packages/salt/states/ssh.py*",
"**/site-packages/salt/states/supervisord.py*",
"**/site-packages/salt/states/sysrc.py*",
"**/site-packages/salt/states/trafficserver.py*",
"**/site-packages/salt/states/tuned.py*",
"**/site-packages/salt/states/vbox.py*",
"**/site-packages/salt/states/virt.py.py*",
"**/site-packages/salt/states/zfs.py*",
"**/site-packages/salt/states/zpool.py*",
]
)
if sys.platform.lower().startswith("darwin"):
dir_patterns.extend(
[
"**/pkgconfig",
"**/share",
"**/artifacts/salt/opt",
"**/artifacts/salt/etc",
"**/artifacts/salt/Lib",
]
)
for root, dirs, files in os.walk(
str(REPO_ROOT / ".nox"), topdown=True, followlinks=False
):
for dirname in dirs:
path = os.path.join(root, dirname)
if not os.path.exists(path):
continue
match_path = pathlib.Path(path).as_posix()
for pattern in dir_patterns:
if fnmatch.fnmatch(str(match_path), pattern):
session.log(f"Deleting directory: {match_path}")
shutil.rmtree(path)
for filename in files:
path = os.path.join(root, filename)
if not os.path.exists(path):
continue
match_path = pathlib.Path(path).as_posix()
for pattern in file_patterns:
if fnmatch.fnmatch(str(match_path), pattern):
session.log(f"Deleting file: {match_path}")
try:
os.remove(path)
except FileNotFoundError:
pass
@nox.session(python="3", name="combine-coverage")

44
tools/vm.py
View file

@ -269,11 +269,12 @@ def test(
if "photonos" in name:
skip_known_failures = os.environ.get("SKIP_INITIAL_PHOTONOS_FAILURES", "1")
env["SKIP_INITIAL_PHOTONOS_FAILURES"] = skip_known_failures
vm.run_nox(
returncode = vm.run_nox(
nox_session=nox_session,
session_args=nox_session_args,
env=env,
)
ctx.exit(returncode)
@vm.command(
@ -329,11 +330,12 @@ def testplan(
if "photonos" in name:
skip_known_failures = os.environ.get("SKIP_INITIAL_PHOTONOS_FAILURES", "1")
env["SKIP_INITIAL_PHOTONOS_FAILURES"] = skip_known_failures
vm.run_nox(
returncode = vm.run_nox(
nox_session=nox_session,
session_args=nox_session_args,
env=env,
)
ctx.exit(returncode)
@vm.command(
@ -357,7 +359,26 @@ def install_dependencies(ctx: Context, name: str, nox_session: str = "ci-test-3"
Install test dependencies on VM.
"""
vm = VM(ctx=ctx, name=name, region_name=ctx.parser.options.region)
vm.install_dependencies(nox_session)
returncode = vm.install_dependencies(nox_session)
ctx.exit(returncode)
@vm.command(
name="pre-archive-cleanup",
arguments={
"name": {
"help": "The VM Name",
"metavar": "VM_NAME",
},
},
)
def pre_archive_cleanup(ctx: Context, name: str):
"""
Pre `.nox` directory compress cleanup.
"""
vm = VM(ctx=ctx, name=name, region_name=ctx.parser.options.region)
returncode = vm.run_nox("pre-archive-cleanup")
ctx.exit(returncode)
@vm.command(
@ -374,7 +395,8 @@ def compress_dependencies(ctx: Context, name: str):
Compress the .nox/ directory in the VM.
"""
vm = VM(ctx=ctx, name=name, region_name=ctx.parser.options.region)
vm.compress_dependencies()
returncode = vm.compress_dependencies()
ctx.exit(returncode)
@vm.command(
@ -391,7 +413,8 @@ def decompress_dependencies(ctx: Context, name: str):
Decompress a dependencies archive into the .nox/ directory in the VM.
"""
vm = VM(ctx=ctx, name=name, region_name=ctx.parser.options.region)
vm.decompress_dependencies()
returncode = vm.decompress_dependencies()
ctx.exit(returncode)
@vm.command(
@ -425,7 +448,8 @@ def combine_coverage(ctx: Context, name: str):
Combine the several code coverage files into a single one in the VM.
"""
vm = VM(ctx=ctx, name=name, region_name=ctx.parser.options.region)
vm.combine_coverage()
returncode = vm.combine_coverage()
ctx.exit(returncode)
@vm.command(
@ -1062,25 +1086,25 @@ class VM:
capture=False,
pseudo_terminal=True,
)
self.ctx.exit(ret.returncode)
return ret.returncode
def combine_coverage(self):
"""
Combine the code coverage databases
"""
self.run_nox("combine-coverage", session_args=[self.name])
return self.run_nox("combine-coverage", session_args=[self.name])
def compress_dependencies(self):
"""
Compress .nox/ into nox.<vm-name>.tar.* in the VM
"""
self.run_nox("compress-dependencies", session_args=[self.name])
return self.run_nox("compress-dependencies", session_args=[self.name])
def decompress_dependencies(self):
"""
Decompress nox.<vm-name>.tar.* if it exists in the VM
"""
self.run_nox("decompress-dependencies", session_args=[self.name])
return self.run_nox("decompress-dependencies", session_args=[self.name])
def download_dependencies(self):
"""