salt/tools/changelog.py

"""
These commands are used manage Salt's changelog.
"""
# pylint: disable=resource-leakage,broad-except
from __future__ import annotations

import datetime
import logging
import os
import pathlib
import re
import subprocess
import sys
import textwrap

from ptscripts import Context, command_group

import tools.utils

REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
CHANGELOG_LIKE_RE = re.compile(r"([\d]+)\.([a-z]+)$")
CHANGELOG_TYPES = (
    "removed",
    "deprecated",
    "changed",
    "fixed",
    "added",
    "security",
)
CHANGELOG_ENTRY_RE = re.compile(
    r"([\d]+|(CVE|cve)-[\d]{{4}}-[\d]+)\.({})(\.md)?$".format("|".join(CHANGELOG_TYPES))
)

log = logging.getLogger(__name__)

# Define the command group
changelog = command_group(
    name="changelog",
    help="Changelog tools",
    description=__doc__,
    venv_config={
        "requirements_files": [
            tools.utils.REPO_ROOT
            / "requirements"
            / "static"
            / "ci"
            / "py{}.{}".format(*sys.version_info)
            / "changelog.txt"
        ],
    },
)


@changelog.command(
    name="pre-commit-checks",
    arguments={
        "files": {
            "nargs": "*",
        }
    },
)
def check_changelog_entries(ctx: Context, files: list[pathlib.Path]):
    """
    Run pre-commit checks on changelog snippets.
    """
    docs_path = REPO_ROOT / "doc"
    tests_integration_files_path = REPO_ROOT / "tests" / "integration" / "files"
    changelog_entries_path = REPO_ROOT / "changelog"
    exitcode = 0
    for entry in files:
        path = pathlib.Path(entry).resolve()
        # Is it under changelog/
        try:
            path.relative_to(changelog_entries_path)
            if path.name in (".keep", ".template.jinja"):
                # This is the file we use so git doesn't delete the changelog/ directory
                continue
            # Is it named properly
            if not CHANGELOG_ENTRY_RE.match(path.name):
                ctx.error(
                    "The changelog entry '{}' should have one of the following extensions: {}.".format(
                        path.relative_to(REPO_ROOT),
                        ", ".join(f"{ext}.md" for ext in CHANGELOG_TYPES),
                    ),
                )
                exitcode = 1
                continue
            if path.suffix != ".md":
                ctx.error(
                    f"Please rename '{path.relative_to(REPO_ROOT)}' to "
                    f"'{path.relative_to(REPO_ROOT)}.md'"
                )
                exitcode = 1
                continue
        except ValueError:
            # No, carry on
            pass
        # Does it look like a changelog entry
        if CHANGELOG_LIKE_RE.match(path.name) and not CHANGELOG_ENTRY_RE.match(
            path.name
        ):
            try:
                # Is this under doc/
                path.relative_to(docs_path)
                # Yes, carry on
                continue
            except ValueError:
                # No, resume the check
                pass
            try:
                # Is this under tests/integration/files
                path.relative_to(tests_integration_files_path)
                # Yes, carry on
                continue
            except ValueError:
                # No, resume the check
                pass
            ctx.error(
                "The changelog entry '{}' should have one of the following extensions: {}.".format(
                    path.relative_to(REPO_ROOT),
                    ", ".join(f"{ext}.md" for ext in CHANGELOG_TYPES),
                )
            )
            exitcode = 1
            continue
        # Is it a changelog entry
        if not CHANGELOG_ENTRY_RE.match(path.name):
            # No? Carry on
            continue
        # Is the changelog entry in the right path?
        try:
            path.relative_to(changelog_entries_path)
        except ValueError:
            exitcode = 1
            ctx.error(
                "The changelog entry '{}' should be placed under '{}/', not '{}'".format(
                    path.name,
                    changelog_entries_path.relative_to(REPO_ROOT),
                    path.relative_to(REPO_ROOT).parent,
                )
            )
        if path.suffix != ".md":
            ctx.error(
                f"Please rename '{path.relative_to(REPO_ROOT)}' to "
                f"'{path.relative_to(REPO_ROOT)}.md'"
            )
            exitcode = 1
    ctx.exit(exitcode)


def _get_changelog_contents(ctx: Context, version: str):
    """
    Return the full changelog generated by towncrier.
    """
    return ctx.run(
        "towncrier",
        "build",
        "--draft",
        f"--version={version}",
        capture=True,
    ).stdout.decode()


def _get_pkg_changelog_contents(ctx: Context, version: str):
    """
    Return a version of the changelog entries suitable for packaged changelogs.
    """
    changes = _get_changelog_contents(ctx, version)
    changes = "\n".join(changes.split("\n")[2:])
    changes = changes.replace(
        textwrap.dedent(
            """
        Removed
        -------

        """
        ),
        "",
    )
    changes = changes.replace(
        textwrap.dedent(
            """
        Deprecated
        ----------

        """
        ),
        "",
    )
    changes = changes.replace(
        textwrap.dedent(
            """
        Changed
        -------

        """
        ),
        "",
    )
    changes = changes.replace(
        textwrap.dedent(
            """
        Fixed
        -----

        """
        ),
        "",
    )
    changes = changes.replace(
        textwrap.dedent(
            """
        Added
        -----

        """
        ),
        "",
    )
    return changes


def _get_salt_version():
    return (
        subprocess.run(
            ["python3", "salt/version.py"], stdout=subprocess.PIPE, check=True
        )
        .stdout.decode()
        .strip()
    )


@changelog.command(
    name="update-rpm",
    arguments={
        "salt_version": {
            "help": (
                "The salt package version. If not passed "
                "it will be discovered by running 'python3 salt/version.py'."
            ),
            "nargs": "?",
            "default": None,
        },
        "draft": {
            "help": "Do not make any changes, instead output what would be changed.",
        },
    },
)
def update_rpm(ctx: Context, salt_version: str, draft: bool = False):
    if salt_version is None:
        salt_version = _get_salt_version()
    changes = _get_pkg_changelog_contents(ctx, salt_version)
    ctx.info("Salt version is %s", salt_version)
    orig = ctx.run(
        "sed",
        f"s/Version: .*/Version: {salt_version}/g",
        "pkg/rpm/salt.spec",
        capture=True,
        check=True,
    ).stdout.decode()
    dt = datetime.datetime.utcnow()
    date = dt.strftime("%a %b %d %Y")
    header = f"* {date} Salt Project Packaging <saltproject-packaging@vmware.com> - {salt_version}\n"
    parts = orig.split("%changelog")
    tmpspec = "pkg/rpm/salt.spec.1"
    with open(tmpspec, "w") as wfp:
        wfp.write(parts[0])
        wfp.write("%changelog\n")
        wfp.write(header)
        wfp.write(changes)
        wfp.write(parts[1])
    try:
        with open(tmpspec) as rfp:
            if draft:
                ctx.info(rfp.read())
            else:
                with open("pkg/rpm/salt.spec", "w") as wfp:
                    wfp.write(rfp.read())
    finally:
        os.remove(tmpspec)


@changelog.command(
    name="update-deb",
    arguments={
        "salt_version": {
            "help": (
                "The salt package version. If not passed "
                "it will be discovered by running 'python3 salt/version.py'."
            ),
            "nargs": "?",
            "default": None,
        },
        "draft": {
            "help": "Do not make any changes, instead output what would be changed.",
        },
    },
)
def update_deb(ctx: Context, salt_version: str, draft: bool = False):
    if salt_version is None:
        salt_version = _get_salt_version()
    changes = _get_pkg_changelog_contents(ctx, salt_version)
    formated = "\n".join([f"  {_.replace('-', '*', 1)}" for _ in changes.split("\n")])
    dt = datetime.datetime.utcnow()
    date = dt.strftime("%a, %d %b %Y %H:%M:%S +0000")
    tmpchanges = "pkg/rpm/salt.spec.1"
    with open(tmpchanges, "w") as wfp:
        wfp.write(f"salt ({salt_version}) stable; urgency=medium\n\n")
        wfp.write(formated)
        wfp.write(
            f"\n -- Salt Project Packaging <saltproject-packaging@vmware.com> {date}\n\n"
        )
        with open("pkg/debian/changelog") as rfp:
            wfp.write(rfp.read())
    try:
        with open(tmpchanges) as rfp:
            if draft:
                ctx.info(rfp.read())
            else:
                with open("pkg/debian/changelog", "w") as wfp:
                    wfp.write(rfp.read())
    finally:
        os.remove(tmpchanges)


@changelog.command(
    name="update-release-notes",
    arguments={
        "salt_version": {
            "help": (
                "The salt version used to generate the release notes. If not passed "
                "it will be discovered by running 'python3 salt/version.py'."
            ),
            "nargs": "?",
            "default": None,
        },
        "draft": {
            "help": "Do not make any changes, instead output what would be changed.",
        },
    },
)
def update_release_notes(ctx: Context, salt_version: str, draft: bool = False):
    if salt_version is None:
        salt_version = _get_salt_version()
    if "+" in salt_version:
        major_version = salt_version.split("+", 1)[0]
    else:
        major_version = salt_version
    changes = _get_changelog_contents(ctx, salt_version)
    changes = "\n".join(changes.split("\n")[2:])
    tmpnotes = f"doc/topics/releases/{salt_version}.md.tmp"
    try:
        with open(f"doc/topics/releases/{major_version}.md") as rfp:
            existing = rfp.read()
    except FileNotFoundError:
        existing = ""
    with open(tmpnotes, "w") as wfp:
        wfp.write(existing)
        wfp.write("\n## Changelog\n")
        wfp.write(changes)
    try:
        with open(tmpnotes) as rfp:
            contents = rfp.read().strip()
            if draft:
                ctx.print(contents, soft_wrap=True)
            else:
                with open(f"doc/topics/releases/{salt_version}.md", "w") as wfp:
                    wfp.write(contents)
    finally:
        os.remove(tmpnotes)


@changelog.command(
    name="update-changelog-md",
    arguments={
        "salt_version": {
            "help": (
                "The salt version to use in the changelog. If not passed "
                "it will be discovered by running 'python3 salt/version.py'."
            ),
            "nargs": "?",
            "default": None,
        },
        "draft": {
            "help": "Do not make any changes, instead output what would be changed.",
        },
    },
)
def generate_changelog_md(ctx: Context, salt_version: str, draft: bool = False):
    if salt_version is None:
        salt_version = _get_salt_version()
    cmd = ["towncrier", "build", f"--version={salt_version}"]
    if draft:
        cmd += ["--draft"]
    else:
        cmd += ["--yes"]
    ctx.run(*cmd, check=True)
    ctx.run("git", "restore", "--staged", "CHANGELOG.md", "changelog/", check=True)