Generate package changelogs

.github/workflows/build-deb-packages.yml

@@ -42,14 +42,18 @@ jobs:
           apt update
           apt install -y python3 python3-venv build-essential devscripts debhelper bash-completion
 
+      - name: Pip Install Tools Requirements
+        run: |
+          pip3 install -r $(pwd)/requirements/static/ci/py3.10/tools.txt
+
       - name: Build Deb
         env:
           SALT_ONEDIR_ARCHIVE: "${{ github.workspace }}/build-deb-pkg/artifacts/salt-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch }}.tar.xz"
         run: |
           cd build-deb-pkg
           echo "${{ inputs.salt-version }}" > salt/_version.txt
+          tools changelog update-deb
           ln -sf pkg/debian/ .
-          sed -i 's/SALT_RELEASE_VERSION/${{ inputs.salt-version }}/g' debian/changelog
           debuild -e SALT_ONEDIR_ARCHIVE -uc -us
 
       - name: Upload DEBs

.github/workflows/build-rpm-packages.yml

@@ -40,12 +40,16 @@ jobs:
           yum -y update
           yum -y install python3 python3-pip openssl git rpmdevtools rpmlint systemd-units libxcrypt-compat
 
+      - name: Pip Install Tools Requirements
+        run: |
+          pip3 install -r $(pwd)/requirements/static/ci/py3.10/tools.txt
+
       - name: Build RPM
         env:
           SALT_ONEDIR_ARCHIVE: "${{ github.workspace }}/artifacts/salt-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch }}.tar.xz"
         run: |
           echo "${{ inputs.salt-version }}" > salt/_version.txt
-          sed -i 's/^Version: \(.*\)$/Version: ${{ inputs.salt-version }}/g' pkg/rpm/salt.spec
+          tools changlog update-rpm
           rpmbuild -bb --define="_salt_src $(pwd)" $(pwd)/pkg/rpm/salt.spec
 
       - name: Upload RPMs

tools/__init__.py

@@ -3,6 +3,7 @@ import logging
 import tools.ci
 import tools.pkg
 import tools.vm
+import tools.changelog
 
 for name in ("boto3", "botocore", "urllib3"):
     logging.getLogger(name).setLevel(logging.INFO)

tools/changelog.py

@@ -0,0 +1,239 @@
+"""
+These commands are used manage Salt's changelog.
+"""
+# pylint: disable=resource-leakage,broad-except
+from __future__ import annotations
+
+import datetime
+import fnmatch
+import logging
+import os
+import pathlib
+import shutil
+import subprocess
+import sys
+import textwrap
+
+import yaml
+from ptscripts import Context, command_group
+
+log = logging.getLogger(__name__)
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
+
+# Define the command group
+cl = command_group(name="changelog", help="Changelog tools", description=__doc__)
+
+
+def changelog(version):
+    """
+    Return the full changelog generated by towncrier.
+    """
+    proc = subprocess.run(["towncrier", "build", "--draft", f"--version={version}"], stdout=subprocess.PIPE)
+    return proc.stdout.decode()
+
+
+def pkg_changelog(version):
+    """
+    Return a version of the changelog entries suitable for packaged changelogs.
+    """
+    changes = changelog(version)
+    changes = "\n".join(changes.split("\n")[2:])
+    changes = changes.replace(textwrap.dedent("""
+        Removed
+        -------
+
+        """), "")
+    changes = changes.replace(textwrap.dedent("""
+        Deprecated
+        ----------
+
+        """), "")
+    changes = changes.replace(textwrap.dedent("""
+        Changed
+        -------
+
+        """), "")
+    changes = changes.replace(textwrap.dedent("""
+        Fixed
+        -----
+
+        """), "")
+    changes = changes.replace(textwrap.dedent("""
+        Added
+        -----
+
+        """), "")
+    return changes
+
+
+def version():
+    proc = subprocess.run(["python3", "-m", "salt.version"], stdout=subprocess.PIPE, check=True)
+    return proc.stdout.decode().strip()
+
+DEFAULT_VERSION = version()
+
+@cl.command(
+    name="update-rpm",
+    arguments={
+        "salt_version": {
+            "help": (
+                "The salt package version. If not passed "
+                "it will be discovered by running 'python3 salt/version.py'."
+            ),
+            "nargs": "?",
+            "default": DEFAULT_VERSION,
+        },
+        "draft": {
+            "help": (
+                "Do not make any changes, instead output what would be changed."
+            ),
+            "type": "bool",
+            "default": False,
+        },
+    },
+)
+def update_rpm(ctx: Context, salt_version: str, draft: bool):
+    proc = subprocess.run(["towncrier", "build", "--draft", f"--version={salt_version}"], stdout=subprocess.PIPE)
+    changes = pgk_changelog(salt_version)
+    proc = subprocess.run(["sed", f"s/Version: .*/Version: {salt_version}/g", "pkg/rpm/salt.spec"], stdout=subprocess.PIPE, check=True)
+    orig = proc.stdout.decode()
+
+    dt = datetime.datetime.utcnow()
+    date = dt.strftime("%a %b %d %Y")
+    header = f"* {date} Salt Project Packaging <saltproject-packaging@vmware.com> - {salt_version}\n"
+    parts = orig.split('%changelog')
+    tmpspec = "pkg/rpm/salt.spec.1"
+    with open(tmpspec, "w") as wfp:
+        wfp.write(parts[0])
+        wfp.write("%changelog\n")
+        wfp.write(header)
+        wfp.write(changes)
+        wfp.write(parts[1])
+    try:
+        with open(tmpspec, "r") as rfp:
+            if draft:
+                ctx.info(rfp.read())
+            else:
+                with open("pkg/rpm/salt.spec", "w") as wfp:
+                    wfp.write(rfp.read())
+    finally:
+        os.remove(tmpspec)
+
+
+@cl.command(
+    name="update-deb",
+    arguments={
+        "salt_version": {
+            "help": (
+                "The salt package version. If not passed "
+                "it will be discovered by running 'python3 salt/version.py'."
+            ),
+            "nargs": "?",
+            "default": DEFAULT_VERSION,
+        },
+        "draft": {
+            "help": (
+                "Do not make any changes, instead output what would be changed."
+            ),
+            "type": "bool",
+            "default": False,
+        },
+    },
+)
+def update_deb(ctx: Context, salt_version: str, draft: bool):
+    changes = pkg_changelog(salt_version)
+    formated = "\n".join([f"  {_.replace('-', '*', 1)}" for _ in changes.split('\n')])
+    dt = datetime.datetime.utcnow()
+    date = dt.strftime("%a, %d %b %Y %H:%M:%S +0000")
+    tmpchanges = "pkg/rpm/salt.spec.1"
+    with open(tmpchanges, "w") as wfp:
+        wfp.write(f"salt ({salt_version}) stable; urgency=medium\n\n")
+        wfp.write(formated)
+        wfp.write(f"\n -- Salt Project Packaging <saltproject-packaging@vmware.com> {date}\n\n")
+        with open("pkg/debian/changelog", "r") as rfp:
+            wfp.write(rfp.read())
+    try:
+        with open(tmpchanges, "r") as rfp:
+            if draft:
+                ctx.info(rfp.read())
+            else:
+                with open("pkg/deb/changelog", "w") as wfp:
+                    wfp.write(rfp.read())
+    finally:
+        os.remove(tmpchanges)
+
+@cl.command(
+    name="update-release-notes",
+    arguments={
+        "salt_version": {
+            "help": (
+                "The salt version used to generate the release notes. If not passed "
+                "it will be discovered by running 'python3 salt/version.py'."
+            ),
+            "nargs": "?",
+            "default": DEFAULT_VERSION,
+        },
+        "draft": {
+            "help": (
+                "Do not make any changes, instead output what would be changed."
+            ),
+            "type": "bool",
+            "default": False,
+        },
+    },
+)
+def update_release_notes(ctx: Context, salt_version: str, draft: bool):
+    if "+" in version:
+        major_version = salt_version.split("+", 1)[0]
+    else:
+        major_vesrion = salt_versionversion
+    changes = changelog(salt_version)
+    changes = "\n".join(changes.split("\n")[2:])
+    tmpnotes = f"doc/topics/releases/{version}.rst.tmp"
+    try:
+        with open(f"doc/topics/releases/{major_version}.rst", "r") as rfp:
+            existing = rfp.read()
+    except FileNotFoundError:
+        existing = ""
+    with open(tmpnotes, "w") as wfp:
+        wfp.write(existing)
+        wfp.write(changes)
+    try:
+        with open(tmpnotes, "r") as rfp:
+            if draft:
+                ctx.info(rfp.read())
+            else:
+                with open(f"doc/topics/releases/{version}.rst", "w") as wfp:
+                    wfp.write(rfp.read())
+    finally:
+        os.remove(tmpchanges)
+
+
+@cl.command(
+    name="generate-changelog-md",
+    arguments={
+        "salt_version": {
+            "help": (
+                "The salt version to use in the changelog. If not passed "
+                "it will be discovered by running 'python3 salt/version.py'."
+            ),
+            "nargs": "?",
+            "default": DEFAULT_VERSION,
+        },
+        "draft": {
+            "help": (
+                "The draft option determines if we should remove the fragment "
+                "files from the changelog directory"
+            ),
+            "type": "bool",
+            "default": True,
+        },
+    },
+)
+def update_release_notes(ctx: Context, salt_version: str, draft: bool):
+    cmd = ["towncrier", "build", "--version={version}"]
+    if draft:
+        cmd += ["--draft"]
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE)
+    return proc.stdout.decode()