Building documentation is again a separate step

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

.github/actionlint.yaml

@@ -8,3 +8,5 @@ self-hosted-runner:
     - repo-nightly
     - repo-staging
     - repo-release
+    - medium
+    - large

.github/workflows/build-docs.yml

@@ -0,0 +1,132 @@
+---
+name: Build Documentation
+
+on:
+  workflow_call:
+    inputs:
+      salt-version:
+        type: string
+        required: true
+        description: The Salt version to set prior to building packages.
+      cache-seed:
+        required: true
+        type: string
+        description: Seed used to invalidate caches
+
+jobs:
+  build:
+    name: Build
+    runs-on:
+      - self-hosted
+      - linux
+      - medium
+      - x86_64
+    strategy:
+      fail-fast: false
+      max-parallel: 6
+      matrix:
+        docs-output:
+          - html
+          - epub
+          # - pdf
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Download Release Patch
+        uses: actions/download-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}.patch
+
+      - name: Configure Git
+        shell: bash
+        run: |
+          git config --global user.name "Salt Project Packaging"
+          git config --global user.email saltproject-packaging@vmware.com
+          git config --global --add safe.directory "$(pwd)"
+
+      - name: Apply Release Patch
+        shell: bash
+        run: |
+          git am --committer-date-is-author-date salt-${{ inputs.salt-version }}.patch
+          rm salt-${{ inputs.salt-version }}.patch
+
+      - name: Get Python Version
+        id: get-python-version
+        run: |
+          PY_VERSION=$(python3 -c "import sys; sys.stdout.write('{}.{}'.format(*sys.version_info))")
+          echo "python-version=$PY_VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Setup Python Tools Scripts
+        uses: ./.github/actions/setup-python-tools-scripts
+
+      - name: Cache Python Tools Virtualenvs
+        id: tools-venvs-dependencies-cache
+        uses: actions/cache@v3
+        with:
+          path: .tools-venvs/
+          key: ${{ inputs.cache-seed }}|${{ github.workflow }}|tools-venvs|${{ steps.get-python-version.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
+
+      - name: Prepare Docs Build
+        run: |
+          git clone https://gitlab.com/saltstack/open/docs/builddocs.git .builddocs
+          sudo mkdir -p /usr/share/fonts/truetype /usr/share/fonts/opentype
+          sudo cp -rfv .builddocs/builddocs/files/fonts/truetype/*.ttf /usr/share/fonts/truetype/
+          sudo cp -rfv .builddocs/builddocs/files/fonts/opentype/*.otf /usr/share/fonts/opentype/
+          sudo fc-cache -f -v
+
+      - name: Build Documentation (HTML)
+        if: ${{ matrix.docs-output == 'html' }}
+        shell: bash
+        env:
+          LATEST_RELEASE: "${{ inputs.salt-version }}"
+          SALT_ON_SALTSTACK: "1"
+        run: |
+          tools docs html --archive salt-${{ inputs.salt-version }}-docs-html.tar.xz
+
+      - name: Upload Built Documentation Archive (HTML)
+        if: ${{ matrix.docs-output == 'html' }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}-docs-html.tar.xz
+          path: salt-${{ inputs.salt-version }}-docs-html.tar.xz
+          retention-days: 7
+          if-no-files-found: error
+
+      - name: Build Documentation (ePUB)
+        if: ${{ matrix.docs-output == 'epub' }}
+        shell: bash
+        env:
+          LATEST_RELEASE: "${{ inputs.salt-version }}"
+          SALT_ON_SALTSTACK: "1"
+        run: |
+          tools docs epub
+          mv doc/_build/epub/Salt.epub doc/_build/epub/salt-${{ inputs.salt-version }}.epub
+
+      - name: Upload Built Documentation Archive (ePub)
+        if: ${{ matrix.docs-output == 'epub' }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}.epub
+          path: doc/_build/epub/salt-${{ inputs.salt-version }}.epub
+          retention-days: 7
+          if-no-files-found: error
+
+      - name: Build Documentation (PDF)
+        if: ${{ matrix.docs-output == 'pdf' }}
+        shell: bash
+        env:
+          LATEST_RELEASE: "${{ inputs.salt-version }}"
+          SALT_ON_SALTSTACK: "1"
+        run: |
+          tools docs pdf
+          mv doc/_build/latex/Salt.pdf doc/_build/latex/salt-${{ inputs.salt-version }}.pdf
+
+      - name: Upload Built Documentation Archive (PDF)
+        if: ${{ matrix.docs-output == 'pdf' }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}.pdf
+          path: doc/_build/latex/Salt-${{ inputs.salt-version }}.pdf
+          retention-days: 7
+          if-no-files-found: error

.github/workflows/ci.yml

@@ -213,24 +213,18 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v4
-        id: setup-python
-        with:
-          python-version: "3.10"
+      - name: Get Python Version
+        id: get-python-version
+        run: |
+          PY_VERSION=$(python3 -c "import sys; sys.stdout.write('{}.{}'.format(*sys.version_info))")
+          echo "python-version=$PY_VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Cache Python Tools Virtualenvs
         id: tools-venvs-dependencies-cache
         uses: actions/cache@v3
         with:
           path: .tools-venvs/
-          key: ${{ needs.prepare-workflow.outputs.cache-seed }}|${{ github.workflow }}|tools-venvs|${{ steps.setup-python.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
-
-      - name: Cache Sphinx Doctrees
-        uses: actions/cache@v3
-        with:
-          path: doc/_build/doctrees/
-          key: ${{ needs.prepare-workflow.outputs.cache-seed }}|${{ github.workflow }}|sphinx-doctrees|${{ steps.setup-python.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
+          key: ${{ needs.prepare-workflow.outputs.cache-seed }}|${{ github.workflow }}|tools-venvs|${{ steps.get-python-version.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
 
       - name: Setup Python Tools Scripts
         uses: ./.github/actions/setup-python-tools-scripts
@@ -259,25 +253,13 @@ jobs:
           tools changelog update-release-notes --draft
           tools changelog update-release-notes
 
-      - name: Build Documentation
-        shell: bash
-        env:
-          LATEST_RELEASE: ${{ needs.prepare-workflow.outputs.salt-version }}
-        run: |
-          tools docs html --no-clean --archive salt-${{ needs.prepare-workflow.outputs.salt-version }}-docs.tar.xz
-
       - name: Generate MAN Pages
         shell: bash
+        env:
+          LATEST_RELEASE: "${{ needs.prepare-workflow.outputs.salt-version }}"
+          SALT_ON_SALTSTACK: "1"
         run: |
-          tools docs man --no-clean
-
-      - name: Upload Built Documentation Archive
-        uses: actions/upload-artifact@v3
-        with:
-          name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-docs.tar.xz
-          path: salt-${{ needs.prepare-workflow.outputs.salt-version }}-docs.tar.xz
-          retention-days: 7
-          if-no-files-found: error
+          tools docs man
 
       - name: Update Changelog
         shell: bash
@@ -322,6 +304,17 @@ jobs:
           retention-days: 7
           if-no-files-found: error
 
+  build-docs:
+    name: Documentation
+    if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-docs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }}
+    needs:
+      - prepare-workflow
+      - build-source-tarball
+    uses: ./.github/workflows/build-docs.yml
+    with:
+      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
+      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+
   build-source-tarball:
     name: Build Source Tarball
     if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-source-tarball'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }}
@@ -751,6 +744,7 @@ jobs:
       - prepare-workflow
       - pre-commit
       - lint
+      - build-docs
       - build-deps-onedir
       - build-salt-onedir
       - build-pkgs

.github/workflows/nightly.yml

@@ -211,24 +211,18 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v4
-        id: setup-python
-        with:
-          python-version: "3.10"
+      - name: Get Python Version
+        id: get-python-version
+        run: |
+          PY_VERSION=$(python3 -c "import sys; sys.stdout.write('{}.{}'.format(*sys.version_info))")
+          echo "python-version=$PY_VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Cache Python Tools Virtualenvs
         id: tools-venvs-dependencies-cache
         uses: actions/cache@v3
         with:
           path: .tools-venvs/
-          key: ${{ needs.prepare-workflow.outputs.cache-seed }}|${{ github.workflow }}|tools-venvs|${{ steps.setup-python.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
-
-      - name: Cache Sphinx Doctrees
-        uses: actions/cache@v3
-        with:
-          path: doc/_build/doctrees/
-          key: ${{ needs.prepare-workflow.outputs.cache-seed }}|${{ github.workflow }}|sphinx-doctrees|${{ steps.setup-python.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
+          key: ${{ needs.prepare-workflow.outputs.cache-seed }}|${{ github.workflow }}|tools-venvs|${{ steps.get-python-version.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
 
       - name: Setup Python Tools Scripts
         uses: ./.github/actions/setup-python-tools-scripts
@@ -257,25 +251,13 @@ jobs:
           tools changelog update-release-notes --draft
           tools changelog update-release-notes
 
-      - name: Build Documentation
-        shell: bash
-        env:
-          LATEST_RELEASE: ${{ needs.prepare-workflow.outputs.salt-version }}
-        run: |
-          tools docs html --no-clean --archive salt-${{ needs.prepare-workflow.outputs.salt-version }}-docs.tar.xz
-
       - name: Generate MAN Pages
         shell: bash
+        env:
+          LATEST_RELEASE: "${{ needs.prepare-workflow.outputs.salt-version }}"
+          SALT_ON_SALTSTACK: "1"
         run: |
-          tools docs man --no-clean
-
-      - name: Upload Built Documentation Archive
-        uses: actions/upload-artifact@v3
-        with:
-          name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-docs.tar.xz
-          path: salt-${{ needs.prepare-workflow.outputs.salt-version }}-docs.tar.xz
-          retention-days: 7
-          if-no-files-found: error
+          tools docs man
 
       - name: Update Changelog
         shell: bash
@@ -320,6 +302,17 @@ jobs:
           retention-days: 7
           if-no-files-found: error
 
+  build-docs:
+    name: Documentation
+    if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-docs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }}
+    needs:
+      - prepare-workflow
+      - build-source-tarball
+    uses: ./.github/workflows/build-docs.yml
+    with:
+      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
+      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+
   build-source-tarball:
     name: Build Source Tarball
     if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-source-tarball'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }}
@@ -796,6 +789,7 @@ jobs:
       - prepare-workflow
       - pre-commit
       - lint
+      - build-docs
       - build-deps-onedir
       - build-salt-onedir
       - build-pkgs

.github/workflows/scheduled.yml

@@ -211,24 +211,18 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v4
-        id: setup-python
-        with:
-          python-version: "3.10"
+      - name: Get Python Version
+        id: get-python-version
+        run: |
+          PY_VERSION=$(python3 -c "import sys; sys.stdout.write('{}.{}'.format(*sys.version_info))")
+          echo "python-version=$PY_VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Cache Python Tools Virtualenvs
         id: tools-venvs-dependencies-cache
         uses: actions/cache@v3
         with:
           path: .tools-venvs/
-          key: ${{ needs.prepare-workflow.outputs.cache-seed }}|${{ github.workflow }}|tools-venvs|${{ steps.setup-python.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
-
-      - name: Cache Sphinx Doctrees
-        uses: actions/cache@v3
-        with:
-          path: doc/_build/doctrees/
-          key: ${{ needs.prepare-workflow.outputs.cache-seed }}|${{ github.workflow }}|sphinx-doctrees|${{ steps.setup-python.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
+          key: ${{ needs.prepare-workflow.outputs.cache-seed }}|${{ github.workflow }}|tools-venvs|${{ steps.get-python-version.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
 
       - name: Setup Python Tools Scripts
         uses: ./.github/actions/setup-python-tools-scripts
@@ -257,25 +251,13 @@ jobs:
           tools changelog update-release-notes --draft
           tools changelog update-release-notes
 
-      - name: Build Documentation
-        shell: bash
-        env:
-          LATEST_RELEASE: ${{ needs.prepare-workflow.outputs.salt-version }}
-        run: |
-          tools docs html --no-clean --archive salt-${{ needs.prepare-workflow.outputs.salt-version }}-docs.tar.xz
-
       - name: Generate MAN Pages
         shell: bash
+        env:
+          LATEST_RELEASE: "${{ needs.prepare-workflow.outputs.salt-version }}"
+          SALT_ON_SALTSTACK: "1"
         run: |
-          tools docs man --no-clean
-
-      - name: Upload Built Documentation Archive
-        uses: actions/upload-artifact@v3
-        with:
-          name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-docs.tar.xz
-          path: salt-${{ needs.prepare-workflow.outputs.salt-version }}-docs.tar.xz
-          retention-days: 7
-          if-no-files-found: error
+          tools docs man
 
       - name: Update Changelog
         shell: bash
@@ -320,6 +302,17 @@ jobs:
           retention-days: 7
           if-no-files-found: error
 
+  build-docs:
+    name: Documentation
+    if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-docs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }}
+    needs:
+      - prepare-workflow
+      - build-source-tarball
+    uses: ./.github/workflows/build-docs.yml
+    with:
+      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
+      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+
   build-source-tarball:
     name: Build Source Tarball
     if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-source-tarball'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }}
@@ -749,6 +742,7 @@ jobs:
       - prepare-workflow
       - pre-commit
       - lint
+      - build-docs
       - build-deps-onedir
       - build-salt-onedir
       - build-pkgs

.github/workflows/templates/ci.yml.jinja

@@ -56,24 +56,18 @@ on:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v4
-        id: setup-python
-        with:
-          python-version: "3.10"
+      - name: Get Python Version
+        id: get-python-version
+        run: |
+          PY_VERSION=$(python3 -c "import sys; sys.stdout.write('{}.{}'.format(*sys.version_info))")
+          echo "python-version=$PY_VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Cache Python Tools Virtualenvs
         id: tools-venvs-dependencies-cache
         uses: actions/cache@v3
         with:
           path: .tools-venvs/
-          key: ${{ needs.prepare-workflow.outputs.cache-seed }}|${{ github.workflow }}|tools-venvs|${{ steps.setup-python.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
-
-      - name: Cache Sphinx Doctrees
-        uses: actions/cache@v3
-        with:
-          path: doc/_build/doctrees/
-          key: ${{ needs.prepare-workflow.outputs.cache-seed }}|${{ github.workflow }}|sphinx-doctrees|${{ steps.setup-python.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
+          key: ${{ needs.prepare-workflow.outputs.cache-seed }}|${{ github.workflow }}|tools-venvs|${{ steps.get-python-version.outputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
 
       - name: Setup Python Tools Scripts
         uses: ./.github/actions/setup-python-tools-scripts
@@ -102,25 +96,13 @@ on:
           tools changelog update-release-notes --draft
           tools changelog update-release-notes
 
-      - name: Build Documentation
-        shell: bash
-        env:
-          LATEST_RELEASE: ${{ needs.prepare-workflow.outputs.salt-version }}
-        run: |
-          tools docs html --no-clean --archive salt-${{ needs.prepare-workflow.outputs.salt-version }}-docs.tar.xz
-
       - name: Generate MAN Pages
         shell: bash
+        env:
+          LATEST_RELEASE: "${{ needs.prepare-workflow.outputs.salt-version }}"
+          SALT_ON_SALTSTACK: "1"
         run: |
-          tools docs man --no-clean
-
-      - name: Upload Built Documentation Archive
-        uses: actions/upload-artifact@v3
-        with:
-          name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-docs.tar.xz
-          path: salt-${{ needs.prepare-workflow.outputs.salt-version }}-docs.tar.xz
-          retention-days: 7
-          if-no-files-found: error
+          tools docs man
 
       - name: Update Changelog
         shell: bash
@@ -168,6 +150,24 @@ on:
   <%- endif %>
 
 
+  <%- set job_name = "build-docs" %>
+  <%- if includes.get(job_name, True) %>
+
+  <{ job_name }>:
+    <%- do conclusion_needs.append(job_name) %>
+    name: Documentation
+    if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['<{ job_name }>'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }}
+    needs:
+      - prepare-workflow
+      - build-source-tarball
+    uses: ./.github/workflows/build-docs.yml
+    with:
+      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
+      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+
+  <%- endif %>
+
+
   <%- set job_name = "build-source-tarball" %>
   <%- if includes.get(job_name, True) %>
 

tools/ci.py

@@ -252,6 +252,7 @@ def define_jobs(ctx: Context, event_name: str, changed_files: pathlib.Path):
         "lint": True,
         "test": True,
         "prepare-release": True,
+        "build-docs": True,
         "build-source-tarball": True,
         "build-deps-onedir": True,
         "build-salt-onedir": True,
@@ -294,6 +295,15 @@ def define_jobs(ctx: Context, event_name: str, changed_files: pathlib.Path):
             wfh.write("De-selecting the 'lint' job.\n")
         jobs["lint"] = False
 
+    required_docs_changes: set[str] = {
+        changed_files_contents["salt"],
+        changed_files_contents["docs"],
+    }
+    if required_docs_changes == {"false"}:
+        with open(github_step_summary, "a", encoding="utf-8") as wfh:
+            wfh.write("De-selecting the 'build-docs' job.\n")
+        jobs["build-docs"] = False
+
     required_test_changes: set[str] = {
         changed_files_contents["testrun"],
         changed_files_contents["golden_images"],

tools/docs.py

@@ -76,7 +76,7 @@ def html(ctx: Context, no_clean: bool = False, archive: pathlib.Path = None):
     arguments={
         "no_clean": {
             "help": "Don't cleanup prior to building",
-        }
+        },
     },
 )
 def epub(ctx: Context, no_clean: bool = False):