mirror of
https://github.com/saltstack/salt.git
synced 2025-04-15 17:20:19 +00:00
Replace tests/unit/test_doc.py with a pre-commit hook.
We don't need to test docs on all platforms. In turn, we make the check a pre-commit hook which seems more suited.
This commit is contained in:
Pedro Algarvio
parent
a5b9961f78
commit
6a4703146c
5 changed files with 476 additions and 601 deletions
|
|
@ -598,3 +598,16 @@ repos:
|
|||
- -e
|
||||
- lint-tests-pre-commit
|
||||
- --
|
||||
|
||||
- repo: https://github.com/saltstack/salt-nox-pre-commit
|
||||
rev: master
|
||||
hooks:
|
||||
- id: nox-py2
|
||||
alias: check-docs
|
||||
name: Check Docs
|
||||
files: ^(salt/.*\.py|doc/ref/.*\.rst)$
|
||||
args:
|
||||
- -e
|
||||
- invoke-pre-commit
|
||||
- --
|
||||
- docs.check
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
from invoke import Collection # pylint: disable=3rd-party-module-not-gated
|
||||
from . import docs
|
||||
|
||||
ns = Collection()
|
||||
docs = Collection.from_module(docs, name="docs")
|
||||
ns.add_collection(docs, name="docs")
|
||||
|
|
|
|||
459
tasks/docs.py
Normal file
459
tasks/docs.py
Normal file
|
|
@ -0,0 +1,459 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
tasks.docstrings
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
Check salt code base for for missing or wrong docstrings
|
||||
"""
|
||||
|
||||
import ast
|
||||
import collections
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
|
||||
from invoke import task # pylint: disable=3rd-party-module-not-gated
|
||||
from tasks import utils
|
||||
|
||||
CODE_DIR = pathlib.Path(__file__).resolve().parent.parent
|
||||
DOCS_DIR = CODE_DIR / "doc"
|
||||
SALT_CODE_DIR = CODE_DIR / "salt"
|
||||
|
||||
os.chdir(str(CODE_DIR))
|
||||
|
||||
python_module_to_doc_path = {}
|
||||
doc_path_to_python_module = {}
|
||||
|
||||
|
||||
check_paths = (
|
||||
"salt/auth",
|
||||
"salt/beacons",
|
||||
"salt/cache",
|
||||
"salt/cloud",
|
||||
"salt/engine",
|
||||
"salt/executors",
|
||||
"salt/fileserver",
|
||||
"salt/grains",
|
||||
"salt/modules",
|
||||
"salt/netapi",
|
||||
"salt/output",
|
||||
"salt/pillar",
|
||||
"salt/proxy",
|
||||
"salt/queues",
|
||||
"salt/renderers",
|
||||
"salt/returners",
|
||||
"salt/roster",
|
||||
"salt/runners",
|
||||
"salt/sdb",
|
||||
"salt/serializers",
|
||||
"salt/states",
|
||||
"salt/thorium",
|
||||
"salt/tokens",
|
||||
"salt/tops",
|
||||
"salt/wheel",
|
||||
)
|
||||
exclude_paths = (
|
||||
"salt/cloud/cli.py",
|
||||
"salt/cloud/exceptions.py",
|
||||
"salt/cloud/libcloudfuncs.py",
|
||||
)
|
||||
|
||||
|
||||
def build_path_cache():
|
||||
"""
|
||||
Build a python module to doc module cache
|
||||
"""
|
||||
|
||||
for path in SALT_CODE_DIR.rglob("*.py"):
|
||||
path = path.resolve().relative_to(CODE_DIR)
|
||||
strpath = str(path)
|
||||
if strpath.endswith("__init__.py"):
|
||||
continue
|
||||
if not strpath.startswith(check_paths):
|
||||
continue
|
||||
if strpath.startswith(exclude_paths):
|
||||
continue
|
||||
|
||||
parts = list(path.parts)
|
||||
stub_path = DOCS_DIR / "ref"
|
||||
# Remove salt from parts
|
||||
parts.pop(0)
|
||||
# Remove the package from parts
|
||||
package = parts.pop(0)
|
||||
# Remove the module from parts
|
||||
module = parts.pop()
|
||||
|
||||
if package == "cloud":
|
||||
package = "clouds"
|
||||
if package == "fileserver":
|
||||
package = "file_server"
|
||||
if package == "netapi":
|
||||
# These are handled differently
|
||||
if not parts:
|
||||
# This is rest_wsgi
|
||||
stub_path = (
|
||||
stub_path
|
||||
/ package
|
||||
/ "all"
|
||||
/ str(path).replace(".py", ".rst").replace(os.sep, ".")
|
||||
)
|
||||
else:
|
||||
# rest_cherrypy, rest_tornado
|
||||
subpackage = parts.pop(0)
|
||||
stub_path = (
|
||||
stub_path
|
||||
/ package
|
||||
/ "all"
|
||||
/ "salt.netapi.{}.rst".format(subpackage)
|
||||
)
|
||||
else:
|
||||
stub_path = (
|
||||
stub_path
|
||||
/ package
|
||||
/ "all"
|
||||
/ str(path).replace(".py", ".rst").replace(os.sep, ".")
|
||||
)
|
||||
stub_path = stub_path.relative_to(CODE_DIR)
|
||||
python_module_to_doc_path[path] = stub_path
|
||||
doc_path_to_python_module[stub_path] = path
|
||||
|
||||
|
||||
build_path_cache()
|
||||
|
||||
|
||||
def build_file_list(files, extension):
|
||||
# Unfortunately invoke does not support nargs.
|
||||
# We migth have been passed --files="foo.py bar.py"
|
||||
# Turn that into a list of paths
|
||||
_files = []
|
||||
for path in files:
|
||||
if not path:
|
||||
continue
|
||||
for spath in path.split():
|
||||
if not spath.endswith(extension):
|
||||
continue
|
||||
_files.append(spath)
|
||||
if not _files:
|
||||
_files = CODE_DIR.rglob("*{}".format(extension))
|
||||
else:
|
||||
_files = [pathlib.Path(fname) for fname in _files]
|
||||
_files = [path.relative_to(CODE_DIR) for path in _files]
|
||||
return _files
|
||||
|
||||
|
||||
def build_python_module_paths(files):
|
||||
_files = []
|
||||
for path in build_file_list(files, ".py"):
|
||||
strpath = str(path)
|
||||
if strpath.endswith("__init__.py"):
|
||||
continue
|
||||
if not strpath.startswith(check_paths):
|
||||
continue
|
||||
if strpath.startswith(exclude_paths):
|
||||
continue
|
||||
_files.append(path)
|
||||
return _files
|
||||
|
||||
|
||||
def build_docs_paths(files):
|
||||
return build_file_list(files, ".rst")
|
||||
|
||||
|
||||
@task(iterable=["files"], positional=["files"])
|
||||
def check_inline_markup(ctx, files):
|
||||
"""
|
||||
Check docstring for :doc: usage
|
||||
|
||||
We should not be using the ``:doc:`` inline markup option when
|
||||
cross-referencing locations. Use ``:ref:`` or ``:mod:`` instead.
|
||||
|
||||
This task checks for reference to ``:doc:`` usage.
|
||||
|
||||
See Issue #12788 for more information.
|
||||
|
||||
https://github.com/saltstack/salt/issues/12788
|
||||
"""
|
||||
# CD into Salt's repo root directory
|
||||
ctx.cd(CODE_DIR)
|
||||
|
||||
files = build_python_module_paths(files)
|
||||
|
||||
exitcode = 0
|
||||
for path in files:
|
||||
module = ast.parse(path.read_text(), filename=str(path))
|
||||
funcdefs = [node for node in module.body if isinstance(node, ast.FunctionDef)]
|
||||
for funcdef in funcdefs:
|
||||
docstring = ast.get_docstring(funcdef, clean=True)
|
||||
if not docstring:
|
||||
continue
|
||||
if ":doc:" in docstring:
|
||||
utils.error(
|
||||
"The {} function in {} contains ':doc:' usage", funcdef.name, path
|
||||
)
|
||||
exitcode += 1
|
||||
utils.exit_invoke(exitcode)
|
||||
|
||||
|
||||
@task(iterable=["files"])
|
||||
def check_stubs(ctx, files):
|
||||
# CD into Salt's repo root directory
|
||||
ctx.cd(CODE_DIR)
|
||||
|
||||
files = build_python_module_paths(files)
|
||||
|
||||
exitcode = 0
|
||||
for path in files:
|
||||
strpath = str(path)
|
||||
if strpath.endswith("__init__.py"):
|
||||
continue
|
||||
if not strpath.startswith(check_paths):
|
||||
continue
|
||||
if strpath.startswith(exclude_paths):
|
||||
continue
|
||||
stub_path = python_module_to_doc_path[path]
|
||||
if not stub_path.exists():
|
||||
exitcode += 1
|
||||
utils.error(
|
||||
"The module at {} does not have a sphinx stub at {}", path, stub_path
|
||||
)
|
||||
utils.exit_invoke(exitcode)
|
||||
|
||||
|
||||
@task(iterable=["files"])
|
||||
def check_virtual(ctx, files):
|
||||
"""
|
||||
Check if .rst files for each module contains the text ".. _virtual"
|
||||
indicating it is a virtual doc page, and, in case a module exists by
|
||||
the same name, it's going to be shaddowed and not accessible
|
||||
"""
|
||||
exitcode = 0
|
||||
files = build_docs_paths(files)
|
||||
for path in files:
|
||||
if path.name == "index.rst":
|
||||
continue
|
||||
contents = path.read_text()
|
||||
if ".. _virtual-" in contents:
|
||||
try:
|
||||
python_module = doc_path_to_python_module[path]
|
||||
utils.error(
|
||||
"The doc file at {} indicates that it's virtual, yet, there's a python module "
|
||||
"at {} that will shaddow it.",
|
||||
path,
|
||||
python_module,
|
||||
)
|
||||
exitcode += 1
|
||||
except KeyError:
|
||||
# This is what we're expecting
|
||||
continue
|
||||
utils.exit_invoke(exitcode)
|
||||
|
||||
|
||||
@task(iterable=["files"])
|
||||
def check_module_indexes(ctx, files):
|
||||
exitcode = 0
|
||||
files = build_docs_paths(files)
|
||||
for path in files:
|
||||
if path.name != "index.rst":
|
||||
continue
|
||||
contents = path.read_text()
|
||||
if ".. autosummary::" not in contents:
|
||||
continue
|
||||
module_index_block = re.search(
|
||||
r"""
|
||||
\.\.\s+autosummary::\s*\n
|
||||
(\s+:[a-z]+:.*\n)*
|
||||
(\s*\n)+
|
||||
(?P<mods>(\s*[a-z0-9_\.]+\s*\n)+)
|
||||
""",
|
||||
contents,
|
||||
flags=re.VERBOSE,
|
||||
)
|
||||
|
||||
if not module_index_block:
|
||||
continue
|
||||
|
||||
module_index = re.findall(
|
||||
r"""\s*([a-z0-9_\.]+)\s*\n""", module_index_block.group("mods")
|
||||
)
|
||||
if module_index != sorted(module_index):
|
||||
exitcode += 1
|
||||
utils.error(
|
||||
"The autosummary mods in {} are not properly sorted. Please sort them.",
|
||||
path,
|
||||
)
|
||||
|
||||
module_index_duplicates = [
|
||||
mod for mod, count in collections.Counter(module_index).items() if count > 1
|
||||
]
|
||||
if module_index_duplicates:
|
||||
exitcode += 1
|
||||
utils.error(
|
||||
"Module index {} contains duplicates: {}", path, module_index_duplicates
|
||||
)
|
||||
# Let's check if all python modules are included in the index
|
||||
path_parts = list(path.parts)
|
||||
# drop doc
|
||||
path_parts.pop(0)
|
||||
# drop ref
|
||||
path_parts.pop(0)
|
||||
# drop "index.rst"
|
||||
path_parts.pop()
|
||||
# drop "all"
|
||||
path_parts.pop()
|
||||
package = path_parts.pop(0)
|
||||
if package == "clouds":
|
||||
package = "cloud"
|
||||
if package == "file_server":
|
||||
package = "fileserver"
|
||||
if package == "configuration":
|
||||
package = "log"
|
||||
path_parts = ["handlers"]
|
||||
python_package = SALT_CODE_DIR.joinpath(package, *path_parts).relative_to(
|
||||
CODE_DIR
|
||||
)
|
||||
modules = set()
|
||||
for module in python_package.rglob("*.py"):
|
||||
if package == "netapi":
|
||||
if module.stem == "__init__":
|
||||
continue
|
||||
if len(module.parts) > 4:
|
||||
continue
|
||||
if len(module.parts) > 3:
|
||||
modules.add(module.parent.stem)
|
||||
else:
|
||||
modules.add(module.stem)
|
||||
elif package == "cloud":
|
||||
if len(module.parts) < 4:
|
||||
continue
|
||||
if module.name == "__init__.py":
|
||||
continue
|
||||
modules.add(module.stem)
|
||||
elif package == "modules":
|
||||
if len(module.parts) > 3:
|
||||
# salt.modules.inspeclib
|
||||
if module.name == "__init__.py":
|
||||
modules.add(module.parent.stem)
|
||||
continue
|
||||
modules.add("{}.{}".format(module.parent.stem, module.stem))
|
||||
continue
|
||||
if module.name == "__init__.py":
|
||||
continue
|
||||
modules.add(module.stem)
|
||||
elif module.name == "__init__.py":
|
||||
continue
|
||||
elif module.name != "__init__.py":
|
||||
modules.add(module.stem)
|
||||
|
||||
missing_modules_in_index = set(modules) - set(module_index)
|
||||
if missing_modules_in_index:
|
||||
exitcode += 1
|
||||
utils.error(
|
||||
"The module index at {} is missing the following modules: {}",
|
||||
path,
|
||||
", ".join(missing_modules_in_index),
|
||||
)
|
||||
extra_modules_in_index = set(module_index) - set(modules)
|
||||
if extra_modules_in_index:
|
||||
exitcode += 1
|
||||
utils.error(
|
||||
"The module index at {} has extra modules(non existing): {}",
|
||||
path,
|
||||
", ".join(extra_modules_in_index),
|
||||
)
|
||||
utils.exit_invoke(exitcode)
|
||||
|
||||
|
||||
@task(iterable=["files"])
|
||||
def check_stray(ctx, files):
|
||||
exitcode = 0
|
||||
exclude_paths = (
|
||||
DOCS_DIR / "_inc",
|
||||
DOCS_DIR / "ref" / "cli" / "_includes",
|
||||
DOCS_DIR / "ref" / "cli",
|
||||
DOCS_DIR / "ref" / "configuration",
|
||||
DOCS_DIR / "ref" / "file_server" / "backends.rst",
|
||||
DOCS_DIR / "ref" / "file_server" / "environments.rst",
|
||||
DOCS_DIR / "ref" / "file_server" / "file_roots.rst",
|
||||
DOCS_DIR / "ref" / "internals",
|
||||
DOCS_DIR / "ref" / "modules" / "all" / "salt.modules.inspectlib.rst",
|
||||
DOCS_DIR / "ref" / "peer.rst",
|
||||
DOCS_DIR / "ref" / "publisheracl.rst",
|
||||
DOCS_DIR / "ref" / "python-api.rst",
|
||||
DOCS_DIR / "ref" / "states" / "aggregate.rst",
|
||||
DOCS_DIR / "ref" / "states" / "altering_states.rst",
|
||||
DOCS_DIR / "ref" / "states" / "backup_mode.rst",
|
||||
DOCS_DIR / "ref" / "states" / "compiler_ordering.rst",
|
||||
DOCS_DIR / "ref" / "states" / "extend.rst",
|
||||
DOCS_DIR / "ref" / "states" / "failhard.rst",
|
||||
DOCS_DIR / "ref" / "states" / "global_state_arguments.rst",
|
||||
DOCS_DIR / "ref" / "states" / "highstate.rst",
|
||||
DOCS_DIR / "ref" / "states" / "include.rst",
|
||||
DOCS_DIR / "ref" / "states" / "layers.rst",
|
||||
DOCS_DIR / "ref" / "states" / "master_side.rst",
|
||||
DOCS_DIR / "ref" / "states" / "ordering.rst",
|
||||
DOCS_DIR / "ref" / "states" / "parallel.rst",
|
||||
DOCS_DIR / "ref" / "states" / "providers.rst",
|
||||
DOCS_DIR / "ref" / "states" / "requisites.rst",
|
||||
DOCS_DIR / "ref" / "states" / "startup.rst",
|
||||
DOCS_DIR / "ref" / "states" / "testing.rst",
|
||||
DOCS_DIR / "ref" / "states" / "top.rst",
|
||||
DOCS_DIR / "ref" / "states" / "vars.rst",
|
||||
DOCS_DIR / "ref" / "states" / "writing.rst",
|
||||
DOCS_DIR / "topics",
|
||||
)
|
||||
exclude_paths = tuple([str(p.relative_to(CODE_DIR)) for p in exclude_paths])
|
||||
files = build_docs_paths(files)
|
||||
for path in files:
|
||||
if not str(path).startswith(str((DOCS_DIR / "ref").relative_to(CODE_DIR))):
|
||||
continue
|
||||
if str(path).startswith(exclude_paths):
|
||||
continue
|
||||
if path.name in ("index.rst", "glossary.rst", "faq.rst", "README.rst"):
|
||||
continue
|
||||
try:
|
||||
python_module = doc_path_to_python_module[path]
|
||||
except KeyError:
|
||||
contents = path.read_text()
|
||||
if ".. _virtual-" in contents:
|
||||
continue
|
||||
exitcode += 1
|
||||
utils.error(
|
||||
"The doc at {} doesn't have a corresponding python module an is considered a stray "
|
||||
"doc. Please remove it.",
|
||||
path,
|
||||
)
|
||||
utils.exit_invoke(exitcode)
|
||||
|
||||
|
||||
@task(iterable=["files"])
|
||||
def check(ctx, files):
|
||||
try:
|
||||
utils.info("Checking inline :doc: markup")
|
||||
check_inline_markup(ctx, files)
|
||||
except SystemExit as exc:
|
||||
if exc.code != 0:
|
||||
raise
|
||||
try:
|
||||
utils.info("Checking python module stubs")
|
||||
check_stubs(ctx, files)
|
||||
except SystemExit as exc:
|
||||
if exc.code != 0:
|
||||
raise
|
||||
try:
|
||||
utils.info("Checking virtual modules")
|
||||
check_virtual(ctx, files)
|
||||
except SystemExit as exc:
|
||||
if exc.code != 0:
|
||||
raise
|
||||
try:
|
||||
utils.info("Checking doc module indexes")
|
||||
check_module_indexes(ctx, files)
|
||||
except SystemExit as exc:
|
||||
if exc.code != 0:
|
||||
raise
|
||||
try:
|
||||
utils.info("Checking stray docs")
|
||||
check_stray(ctx, files)
|
||||
except SystemExit as exc:
|
||||
if exc.code != 0:
|
||||
raise
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
"""
|
||||
tasks.utils
|
||||
~~~~~~~~~~~
|
||||
|
||||
Invoke utilities
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -1,601 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
tests.unit.doc_test
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
"""
|
||||
|
||||
# Import Python libs
|
||||
from __future__ import absolute_import
|
||||
|
||||
import collections
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
# Import Salt libs
|
||||
import salt.modules.cmdmod
|
||||
import salt.utils.files
|
||||
import salt.utils.platform
|
||||
from tests.support.runtests import RUNTIME_VARS
|
||||
|
||||
# Import Salt Testing libs
|
||||
from tests.support.unit import TestCase, skipIf
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DocTestCase(TestCase):
|
||||
"""
|
||||
Unit test case for testing doc files and strings.
|
||||
"""
|
||||
|
||||
@skipIf(True, "SLOWTEST skip")
|
||||
def test_check_for_doc_inline_markup(self):
|
||||
"""
|
||||
We should not be using the ``:doc:`` inline markup option when
|
||||
cross-referencing locations. Use ``:ref:`` or ``:mod:`` instead.
|
||||
|
||||
This test checks for reference to ``:doc:`` usage.
|
||||
|
||||
See Issue #12788 for more information.
|
||||
|
||||
https://github.com/saltstack/salt/issues/12788
|
||||
"""
|
||||
salt_dir = RUNTIME_VARS.CODE_DIR
|
||||
|
||||
if salt.utils.platform.is_windows():
|
||||
if salt.utils.path.which("bash"):
|
||||
# Use grep from git-bash when it exists.
|
||||
cmd = "bash -c 'grep -r :doc: ./salt/"
|
||||
grep_call = salt.modules.cmdmod.run_stdout(cmd=cmd, cwd=salt_dir).split(
|
||||
os.linesep
|
||||
)
|
||||
os_sep = "/"
|
||||
else:
|
||||
# No grep in Windows, use findstr
|
||||
# findstr in windows doesn't prepend 'Binary` to binary files, so
|
||||
# use the '/P' switch to skip files with unprintable characters
|
||||
cmd = 'findstr /C:":doc:" /S /P {0}\\*'.format(salt_dir)
|
||||
grep_call = salt.modules.cmdmod.run_stdout(cmd=cmd).split(os.linesep)
|
||||
os_sep = os.sep
|
||||
else:
|
||||
salt_dir += "/"
|
||||
cmd = "grep -r :doc: " + salt_dir
|
||||
grep_call = salt.modules.cmdmod.run_stdout(cmd=cmd).split(os.linesep)
|
||||
os_sep = os.sep
|
||||
|
||||
test_ret = {}
|
||||
for line in grep_call:
|
||||
# Skip any .pyc files that may be present
|
||||
if line.startswith("Binary"):
|
||||
continue
|
||||
|
||||
# Only split on colons not followed by a '\' as is the case with
|
||||
# Windows Drives
|
||||
regex = re.compile(r":(?!\\)")
|
||||
try:
|
||||
key, val = regex.split(line, 1)
|
||||
except ValueError:
|
||||
log.error("Could not split line: %s", line)
|
||||
continue
|
||||
|
||||
# Don't test man pages, this file, the tox or nox virtualenv files,
|
||||
# the page that documents to not use ":doc:", the doc/conf.py file
|
||||
# or the artifacts directory on nox CI test runs
|
||||
if (
|
||||
"man" in key
|
||||
or ".tox{}".format(os_sep) in key
|
||||
or ".nox{}".format(os_sep) in key
|
||||
or "ext{}".format(os_sep) in key
|
||||
or "artifacts{}".format(os_sep) in key
|
||||
or key.endswith("test_doc.py")
|
||||
or key.endswith(os_sep.join(["doc", "conf.py"]))
|
||||
or key.endswith(os_sep.join(["conventions", "documentation.rst"]))
|
||||
or key.endswith(
|
||||
os_sep.join(["doc", "topics", "releases", "2016.11.2.rst"])
|
||||
)
|
||||
or key.endswith(
|
||||
os_sep.join(["doc", "topics", "releases", "2016.11.3.rst"])
|
||||
)
|
||||
or key.endswith(
|
||||
os_sep.join(["doc", "topics", "releases", "2016.3.5.rst"])
|
||||
)
|
||||
):
|
||||
continue
|
||||
|
||||
# Set up test return dict
|
||||
if test_ret.get(key) is None:
|
||||
test_ret[key] = [val.strip()]
|
||||
else:
|
||||
test_ret[key].append(val.strip())
|
||||
|
||||
# Allow test results to show files with :doc: ref, rather than truncating
|
||||
self.maxDiff = None
|
||||
|
||||
# test_ret should be empty, otherwise there are :doc: references present
|
||||
self.assertEqual(test_ret, {})
|
||||
|
||||
def _check_doc_files(self, module_skip, module_dir, doc_skip, module_doc_dir):
|
||||
"""
|
||||
Ensure various salt modules have associated documentation
|
||||
"""
|
||||
|
||||
salt_dir = RUNTIME_VARS.CODE_DIR
|
||||
|
||||
# Build list of module files
|
||||
module_files = []
|
||||
skip_module_files = module_skip
|
||||
full_module_dir = os.path.join(salt_dir, *module_dir)
|
||||
for file in os.listdir(full_module_dir):
|
||||
if file.endswith(".py"):
|
||||
module_name = os.path.splitext(file)[0]
|
||||
if module_name not in skip_module_files:
|
||||
module_files.append(module_name)
|
||||
# Capture modules in subdirectories like inspectlib and rest_cherrypy
|
||||
elif (
|
||||
os.path.isdir(os.path.join(full_module_dir, file))
|
||||
and not file.startswith("_")
|
||||
and os.path.isfile(os.path.join(full_module_dir, file, "__init__.py"))
|
||||
):
|
||||
module_name = file
|
||||
if module_name not in skip_module_files:
|
||||
module_files.append(module_name)
|
||||
|
||||
# Build list of documentation files
|
||||
module_docs = []
|
||||
skip_doc_files = doc_skip
|
||||
full_module_doc_dir = os.path.join(salt_dir, *module_doc_dir)
|
||||
doc_prefix = ".".join(module_dir) + "."
|
||||
for file in os.listdir(full_module_doc_dir):
|
||||
if file.endswith(".rst"):
|
||||
doc_name = os.path.splitext(file)[0]
|
||||
if doc_name.startswith(doc_prefix):
|
||||
doc_name = doc_name[len(doc_prefix) :]
|
||||
if doc_name not in skip_doc_files:
|
||||
module_docs.append(doc_name)
|
||||
|
||||
module_index_file = os.path.join(full_module_doc_dir, "index.rst")
|
||||
with salt.utils.files.fopen(module_index_file, "rb") as fp:
|
||||
module_index_contents = fp.read().decode("utf-8")
|
||||
|
||||
module_index_block = re.search(
|
||||
r"""
|
||||
\.\.\s+autosummary::\s*\n
|
||||
(\s+:[a-z]+:.*\n)*
|
||||
(\s*\n)+
|
||||
(?P<mods>(\s*[a-z0-9_\.]+\s*\n)+)
|
||||
""",
|
||||
module_index_contents,
|
||||
flags=re.VERBOSE,
|
||||
)
|
||||
|
||||
module_index = re.findall(
|
||||
r"""\s*([a-z0-9_\.]+)\s*\n""", module_index_block.group("mods")
|
||||
)
|
||||
|
||||
# Check that every module has associated documentation file
|
||||
for module in module_files:
|
||||
self.assertIn(
|
||||
module,
|
||||
module_docs,
|
||||
"module file {0} is missing documentation in {1}".format(
|
||||
module, full_module_doc_dir
|
||||
),
|
||||
)
|
||||
|
||||
# Check that every module is listed in the index file
|
||||
self.assertIn(
|
||||
module,
|
||||
module_index,
|
||||
"module file {0} is missing in {1}".format(module, module_index_file),
|
||||
)
|
||||
|
||||
# Check if .rst file for this module contains the text
|
||||
# ".. _virtual" indicating it is a virtual doc page
|
||||
full_module_doc_name = os.path.join(
|
||||
full_module_doc_dir, doc_prefix + module + ".rst"
|
||||
)
|
||||
with salt.utils.files.fopen(full_module_doc_name) as rst_file:
|
||||
rst_text = rst_file.read()
|
||||
virtual_string = 'module file "{0}" is also a virtual doc page {1} and is not accessible'
|
||||
self.assertNotIn(
|
||||
".. _virtual",
|
||||
rst_text,
|
||||
virtual_string.format(module, doc_prefix + module + ".rst"),
|
||||
)
|
||||
|
||||
for doc_file in module_docs:
|
||||
self.assertIn(
|
||||
doc_file,
|
||||
module_files,
|
||||
"Doc file {0} is missing associated module in {1}".format(
|
||||
doc_file, full_module_dir
|
||||
),
|
||||
)
|
||||
# Check that a module index is sorted
|
||||
sorted_module_index = sorted(module_index)
|
||||
self.assertEqual(
|
||||
module_index,
|
||||
sorted_module_index,
|
||||
msg="Module index is not sorted: {}".format(module_index_file),
|
||||
)
|
||||
|
||||
# Check for duplicates inside of a module index
|
||||
module_index_duplicates = [
|
||||
mod for mod, count in collections.Counter(module_index).items() if count > 1
|
||||
]
|
||||
if module_index_duplicates:
|
||||
self.fail(
|
||||
"Module index {0} contains duplicates: {1}".format(
|
||||
module_index_file, module_index_duplicates
|
||||
)
|
||||
)
|
||||
|
||||
# Check for stray module docs
|
||||
# Do not check files listed in doc_skip
|
||||
stray_modules = set(module_index).difference(module_files + doc_skip)
|
||||
if stray_modules:
|
||||
self.fail(
|
||||
"Stray module names {0} in the doc index {1}".format(
|
||||
sorted(list(stray_modules)), module_index_file
|
||||
)
|
||||
)
|
||||
stray_modules = set(module_docs).difference(module_files)
|
||||
if stray_modules:
|
||||
self.fail(
|
||||
"Stray module doc files {0} in the doc folder {1}".format(
|
||||
sorted(list(stray_modules)), full_module_doc_dir
|
||||
)
|
||||
)
|
||||
|
||||
def test_auth_doc_files(self):
|
||||
"""
|
||||
Ensure auth modules have associated documentation
|
||||
|
||||
doc example: doc/ref/auth/all/salt.auth.rest.rst
|
||||
auth module example: salt/auth/rest.py
|
||||
"""
|
||||
|
||||
skip_files = ["__init__"]
|
||||
module_dir = ["salt", "auth"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "auth", "all"]
|
||||
self._check_doc_files(skip_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_beacon_doc_files(self):
|
||||
"""
|
||||
Ensure beacon modules have associated documentation
|
||||
|
||||
doc example: doc/ref/beacons/all/salt.beacon.rest.rst
|
||||
beacon module example: salt/beacons/rest.py
|
||||
"""
|
||||
|
||||
skip_files = ["__init__"]
|
||||
module_dir = ["salt", "beacons"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "beacons", "all"]
|
||||
self._check_doc_files(skip_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_cache_doc_files(self):
|
||||
"""
|
||||
Ensure cache modules have associated documentation
|
||||
|
||||
doc example: doc/ref/cache/all/salt.cache.consul.rst
|
||||
cache module example: salt/cache/consul.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "cache"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "cache", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_cloud_doc_files(self):
|
||||
"""
|
||||
Ensure cloud modules have associated documentation
|
||||
|
||||
doc example: doc/ref/clouds/all/salt.cloud.gce.rst
|
||||
cloud module example: salt/cloud/clouds/gce.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "cloud", "clouds"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "clouds", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_engine_doc_files(self):
|
||||
"""
|
||||
Ensure engine modules have associated documentation
|
||||
|
||||
doc example: doc/ref/engines/all/salt.engines.docker_events.rst
|
||||
engine module example: salt/engines/docker_events.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "engines"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "engines", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_executors_doc_files(self):
|
||||
"""
|
||||
Ensure executor modules have associated documentation
|
||||
|
||||
doc example: doc/ref/executors/all/salt.executors.docker.rst
|
||||
engine module example: salt/executors/docker.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "executors"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "executors", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_fileserver_doc_files(self):
|
||||
"""
|
||||
Ensure fileserver modules have associated documentation
|
||||
|
||||
doc example: doc/ref/fileserver/all/salt.fileserver.gitfs.rst
|
||||
module example: salt/fileserver/gitfs.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "fileserver"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "file_server", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_grain_doc_files(self):
|
||||
"""
|
||||
Ensure grain modules have associated documentation
|
||||
|
||||
doc example: doc/ref/grains/all/salt.grains.core.rst
|
||||
module example: salt/grains/core.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "grains"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "grains", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_module_doc_files(self):
|
||||
"""
|
||||
Ensure modules have associated documentation
|
||||
|
||||
doc example: doc/ref/modules/all/salt.modules.zabbix.rst
|
||||
execution module example: salt/modules/zabbix.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "modules"]
|
||||
skip_doc_files = [
|
||||
"index",
|
||||
"group",
|
||||
"inspectlib.collector",
|
||||
"inspectlib.dbhandle",
|
||||
"inspectlib.entities",
|
||||
"inspectlib.exceptions",
|
||||
"inspectlib.fsdb",
|
||||
"inspectlib.kiwiproc",
|
||||
"inspectlib.query",
|
||||
"kernelpkg",
|
||||
"pkg",
|
||||
"user",
|
||||
"service",
|
||||
"shadow",
|
||||
"sysctl",
|
||||
]
|
||||
doc_dir = ["doc", "ref", "modules", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_netapi_doc_files(self):
|
||||
"""
|
||||
Ensure netapi modules have associated documentation
|
||||
|
||||
doc example: doc/ref/netapi/all/salt.netapi.rest_cherrypy.rst
|
||||
module example: salt/netapi/rest_cherrypy
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "netapi"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "netapi", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_output_doc_files(self):
|
||||
"""
|
||||
Ensure output modules have associated documentation
|
||||
|
||||
doc example: doc/ref/output/all/salt.output.highstate.rst
|
||||
module example: salt/output/highstate.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "output"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "output", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_pillar_doc_files(self):
|
||||
"""
|
||||
Ensure pillar modules have associated documentation
|
||||
|
||||
doc example: doc/ref/pillar/all/salt.pillar.cobbler.rst
|
||||
module example: salt/pillar/cobbler.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "pillar"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "pillar", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_proxy_doc_files(self):
|
||||
"""
|
||||
Ensure proxy modules have associated documentation
|
||||
|
||||
doc example: doc/ref/proxy/all/salt.proxy.docker.rst
|
||||
module example: salt/proxy/docker.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "proxy"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "proxy", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_queues_doc_files(self):
|
||||
"""
|
||||
Ensure queue modules have associated documentation
|
||||
|
||||
doc example: doc/ref/queues/all/salt.queues.sqlite_queue.rst
|
||||
module example: salt/queues/sqlite_queue.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "queues"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "queues", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_renderers_doc_files(self):
|
||||
"""
|
||||
Ensure render modules have associated documentation
|
||||
|
||||
doc example: doc/ref/renderers/all/salt.renderers.json.rst
|
||||
module example: salt/renderers/json.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "renderers"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "renderers", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_returners_doc_files(self):
|
||||
"""
|
||||
Ensure return modules have associated documentation
|
||||
|
||||
doc example: doc/ref/returners/all/salt.returners.cassandra_return.rst
|
||||
module example: salt/returners/cassandra_return.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "returners"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "returners", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_runners_doc_files(self):
|
||||
"""
|
||||
Ensure runner modules have associated documentation
|
||||
|
||||
doc example: doc/ref/runners/all/salt.runners.auth.rst
|
||||
module example: salt/runners/auth.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "runners"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "runners", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_sdb_doc_files(self):
|
||||
"""
|
||||
Ensure sdb modules have associated documentation
|
||||
|
||||
doc example: doc/ref/sdb/all/salt.sdb.rest.rst
|
||||
module example: salt/sdb/rest.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "sdb"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "sdb", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_serializers_doc_files(self):
|
||||
"""
|
||||
Ensure serializer modules have associated documentation
|
||||
|
||||
doc example: doc/ref/serializers/all/salt.serializers.yaml.rst
|
||||
module example: salt/serializers/yaml.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "serializers"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "serializers", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_states_doc_files(self):
|
||||
"""
|
||||
Ensure states have associated documentation
|
||||
|
||||
doc example: doc/ref/states/all/salt.states.zabbix_host.rst
|
||||
module example: salt/states/zabbix_host.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "states"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "states", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_thorium_doc_files(self):
|
||||
"""
|
||||
Ensure thorium modules have associated documentation
|
||||
|
||||
doc example: doc/ref/thorium/all/salt.thorium.calc.rst
|
||||
module example: salt/thorium/calc.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "thorium"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "thorium", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_token_doc_files(self):
|
||||
"""
|
||||
Ensure token modules have associated documentation
|
||||
|
||||
doc example: doc/ref/tokens/all/salt.tokens.localfs.rst
|
||||
module example: salt/tokens/localfs.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "tokens"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "tokens", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_tops_doc_files(self):
|
||||
"""
|
||||
Ensure top modules have associated documentation
|
||||
|
||||
doc example: doc/ref/tops/all/salt.tops.saltclass.rst
|
||||
module example: salt/tops/saltclass.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "tops"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "tops", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
|
||||
def test_wheel_doc_files(self):
|
||||
"""
|
||||
Ensure wheel modules have associated documentation
|
||||
|
||||
doc example: doc/ref/wheel/all/salt.wheel.key.rst
|
||||
module example: salt/wheel/key.py
|
||||
"""
|
||||
|
||||
skip_module_files = ["__init__"]
|
||||
module_dir = ["salt", "wheel"]
|
||||
skip_doc_files = ["index", "all"]
|
||||
doc_dir = ["doc", "ref", "wheel", "all"]
|
||||
self._check_doc_files(skip_module_files, module_dir, skip_doc_files, doc_dir)
|
||||
Loading…
Add table
Reference in a new issue