saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Use PyTest fixtures for the Git Pillar tests

Browse source
This commit is contained in:
Pedro Algarvio 2020-09-02 12:38:30 +01:00
parent b2287e4a23
commit 3ab6b9dca2
No known key found for this signature in database
GPG key ID: BB36BF6584A298FF
5 changed files with 414 additions and 478 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
tests/integration/files/file/base/git_pillar/http/nginx.sls Normal file
View file

@ -0,0 +1,14 @@
{%- set config_dir = pillar['git_pillar']['config_dir'] %}
{{ config_dir }}/nginx.conf:
file.managed:
- source: salt://git_pillar/http/files/nginx.conf
- user: root
{%- if grains['os_family'] == 'FreeBSD' %}
- group: wheel
{%- else %}
- group: root
{%- endif %}
- mode: 644
- makedirs: True
- template: jinja

31
tests/integration/files/file/base/git_pillar/http/init.sls → tests/integration/files/file/base/git_pillar/http/uwsgi.sls
View file

@ -3,28 +3,15 @@
{%- set venv_dir = pillar['git_pillar']['venv_dir'] %}
{%- set root_dir = pillar['git_pillar']['root_dir'] %}
{{ config_dir }}/nginx.conf:
file.managed:
- source: salt://git_pillar/http/files/nginx.conf
- user: root
{% if grains['os_family'] == 'FreeBSD' %}
- group: wheel
{% else %}
- group: root
{% endif %}
- mode: 644
- makedirs: True
- template: jinja
{{ config_dir }}/uwsgi.yml:
file.managed:
- source: salt://git_pillar/http/files/uwsgi.yml
- user: root
{% if grains['os_family'] == 'FreeBSD' %}
{%- if grains['os_family'] == 'FreeBSD' %}
- group: wheel
{% else %}
{%- else %}
- group: root
{% endif %}
{%- endif %}
- mode: 644
- makedirs: True
- template: jinja
@ -32,22 +19,22 @@
{{ root_dir }}:
file.directory:
- user: root
{% if grains['os_family'] == 'FreeBSD' %}
{%- if grains['os_family'] == 'FreeBSD' %}
- group: wheel
{% else %}
{%- else %}
- group: root
{% endif %}
{%- endif %}
- mode: 755
{{ git_dir }}/users:
file.managed:
- source: salt://git_pillar/http/files/users
- user: root
{% if grains['os_family'] == 'FreeBSD' %}
{%- if grains['os_family'] == 'FreeBSD' %}
- group: wheel
{% else %}
{%- else %}
- group: root
{% endif %}
{%- endif %}
- makedirs: True
- mode: 644

107
tests/integration/pillar/test_git_pillar.py
View file

@ -63,15 +63,12 @@ https://github.com/git/git/commit/6bc0cb5
https://github.com/unbit/uwsgi/commit/ac1e354
"""
# Import Python libs
import random
import string
# Import Salt libs
import pytest
import salt.utils.path
import salt.utils.platform
from salt.ext.six.moves import range # pylint: disable=redefined-builtin
from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES as VIRTUALENV_NAMES
from salt.utils.gitfs import (
GITPYTHON_MINVER,
@ -81,13 +78,14 @@ from salt.utils.gitfs import (
PYGIT2_MINVER,
PYGIT2_VERSION,
)
# Import Salt Testing libs
from tests.support.gitfs import (
from tests.support.gitfs import ( # pylint: disable=unused-import
PASSWORD,
USERNAME,
GitPillarHTTPTestBase,
GitPillarSSHTestBase,
ssh_pillar_tests_prep,
webserver_pillar_tests_prep,
webserver_pillar_tests_prep_authenticated,
)
from tests.support.helpers import destructiveTest, skip_if_not_root, slowTest
from tests.support.unit import skipIf
@ -591,6 +589,7 @@ class GitPythonMixin:
},
)
@slowTest
def test_fallback(self):
"""
Test fallback parameter.
@ -632,6 +631,7 @@ class GitPythonMixin:
@skip_if_not_root
@skipIf(not HAS_GITPYTHON, "GitPython >= {} required".format(GITPYTHON_MINVER))
@skipIf(not HAS_SSHD, "sshd not present")
@pytest.mark.usefixtures("ssh_pillar_tests_prep")
class TestGitPythonSSH(GitPillarSSHTestBase, GitPythonMixin):
"""
Test git_pillar with GitPython using SSH authentication
@ -648,6 +648,7 @@ class TestGitPythonSSH(GitPillarSSHTestBase, GitPythonMixin):
@skipIf(not HAS_GITPYTHON, "GitPython >= {} required".format(GITPYTHON_MINVER))
@skipIf(not HAS_NGINX, "nginx not present")
@skipIf(not HAS_VIRTUALENV, "virtualenv not present")
@pytest.mark.usefixtures("webserver_pillar_tests_prep")
class TestGitPythonHTTP(GitPillarHTTPTestBase, GitPythonMixin):
"""
Test git_pillar with GitPython using unauthenticated HTTP
@ -659,6 +660,7 @@ class TestGitPythonHTTP(GitPillarHTTPTestBase, GitPythonMixin):
@skipIf(not HAS_GITPYTHON, "GitPython >= {} required".format(GITPYTHON_MINVER))
@skipIf(not HAS_NGINX, "nginx not present")
@skipIf(not HAS_VIRTUALENV, "virtualenv not present")
@pytest.mark.usefixtures("webserver_pillar_tests_prep_authenticated")
class TestGitPythonAuthenticatedHTTP(TestGitPythonHTTP, GitPythonMixin):
"""
Test git_pillar with GitPython using authenticated HTTP
@ -667,25 +669,6 @@ class TestGitPythonAuthenticatedHTTP(TestGitPythonHTTP, GitPythonMixin):
username = USERNAME
password = PASSWORD
@classmethod
def setUpClass(cls):
"""
Create start the webserver
"""
super().setUpClass()
# Override the URL set up in the parent class to encode the
# username/password into it.
cls.url = "http://{username}:{password}@127.0.0.1:{port}/repo.git".format(
username=cls.username, password=cls.password, port=cls.nginx_port
)
cls.url_extra_repo = "http://{username}:{password}@127.0.0.1:{port}/extra_repo.git".format(
username=cls.username, password=cls.password, port=cls.nginx_port
)
cls.ext_opts["url"] = cls.url
cls.ext_opts["url_extra_repo"] = cls.url_extra_repo
cls.ext_opts["username"] = cls.username
cls.ext_opts["password"] = cls.password
@destructiveTest
@skipIf(_windows_or_mac(), "minion is windows or mac")
@ -695,6 +678,7 @@ class TestGitPythonAuthenticatedHTTP(TestGitPythonHTTP, GitPythonMixin):
"pygit2 >= {} and libgit2 >= {} required".format(PYGIT2_MINVER, LIBGIT2_MINVER),
)
@skipIf(not HAS_SSHD, "sshd not present")
@pytest.mark.usefixtures("ssh_pillar_tests_prep")
class TestPygit2SSH(GitPillarSSHTestBase):
"""
Test git_pillar with pygit2 using SSH authentication
@ -2051,6 +2035,7 @@ class TestPygit2SSH(GitPillarSSHTestBase):
)
self.assertEqual(ret, expected)
@slowTest
def test_fallback(self):
"""
Test fallback parameter.
@ -2170,6 +2155,7 @@ class TestPygit2SSH(GitPillarSSHTestBase):
)
@skipIf(not HAS_NGINX, "nginx not present")
@skipIf(not HAS_VIRTUALENV, "virtualenv not present")
@pytest.mark.usefixtures("webserver_pillar_tests_prep")
class TestPygit2HTTP(GitPillarHTTPTestBase):
"""
Test git_pillar with pygit2 using SSH authentication
@ -2634,6 +2620,7 @@ class TestPygit2HTTP(GitPillarHTTPTestBase):
},
)
@slowTest
def test_fallback(self):
"""
Test fallback parameter.
@ -2678,6 +2665,7 @@ class TestPygit2HTTP(GitPillarHTTPTestBase):
)
@skipIf(not HAS_NGINX, "nginx not present")
@skipIf(not HAS_VIRTUALENV, "virtualenv not present")
@pytest.mark.usefixtures("webserver_pillar_tests_prep_authenticated")
class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
"""
Test git_pillar with pygit2 using SSH authentication
@ -2686,7 +2674,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
possible) in the TestGitPythonSSH class.
"""
user = USERNAME
username = USERNAME
password = PASSWORD
@slowTest
@ -2710,7 +2698,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -2733,7 +2721,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
ext_pillar:
- git:
- master {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
"""
@ -2766,7 +2754,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -2792,11 +2780,11 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
ext_pillar:
- git:
- master {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
- dev {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
"""
@ -2829,7 +2817,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -2855,11 +2843,11 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
ext_pillar:
- git:
- dev {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
- master {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
"""
@ -2892,7 +2880,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -2918,11 +2906,11 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
ext_pillar:
- git:
- master {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
- dev {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
"""
@ -2955,7 +2943,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -2981,11 +2969,11 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
ext_pillar:
- git:
- dev {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
- master {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
"""
@ -3013,7 +3001,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -3039,11 +3027,11 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
ext_pillar:
- git:
- master {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
- dev {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
"""
@ -3074,7 +3062,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -3100,7 +3088,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -3127,7 +3115,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -3154,11 +3142,11 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
- git:
- master {url}:
- root: subdir
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
- top_only {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
- env: base
@ -3196,7 +3184,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_includes: False
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -3222,11 +3210,11 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
ext_pillar:
- git:
- master {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
- top_only {url}:
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
- env: base
@ -3248,7 +3236,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -3257,13 +3245,13 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
- git:
- __env__ {url_extra_repo}:
- name: gitinfo
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
- __env__ {url}:
- name: webinfo
- mountpoint: nowhere
- user: {user}
- user: {username}
- password: {password}
- insecure_auth: True
"""
@ -3294,7 +3282,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -3320,7 +3308,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -3358,7 +3346,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}
@ -3385,6 +3373,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
},
)
@slowTest
def test_fallback(self):
"""
Test fallback parameter.
@ -3394,7 +3383,7 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase):
file_ignore_regex: []
file_ignore_glob: []
git_pillar_provider: pygit2
git_pillar_user: {user}
git_pillar_user: {username}
git_pillar_password: {password}
git_pillar_insecure_auth: True
cachedir: {cachedir}

731
tests/support/gitfs.py
View file

@ -2,92 +2,150 @@
Base classes for gitfs/git_pillar integration tests
"""
import copy
import errno
import logging
import os
import pathlib
import pprint
import shutil
import tempfile
import textwrap
import attr # pylint: disable=3rd-party-module-not-gated
import pytest
import salt.utils.files
import salt.utils.path
import salt.utils.platform
import salt.utils.yaml
from salt.fileserver import gitfs
from salt.pillar import git_pillar
from salt.utils.immutabletypes import freeze
from saltfactories.utils.ports import get_unused_localhost_port
from saltfactories.utils.processes.bases import FactoryDaemonScriptBase
from saltfactories.utils.processes.helpers import start_daemon, terminate_process
from saltfactories.utils.processes.sshd import SshdDaemon
from tests.support.case import ModuleCase
from tests.support.helpers import patched_environ, requires_system_grains
from tests.support.mixins import (
AdaptedConfigurationTestCaseMixin,
LoaderModuleMockMixin,
SaltReturnAssertsMixin,
from tests.support.helpers import (
SKIP_IF_NOT_RUNNING_PYTEST,
patched_environ,
requires_system_grains,
)
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.mock import patch
from tests.support.runtests import RUNTIME_VARS
from tests.support.unit import SkipTest
try:
import psutil
except ImportError:
pass
from tests.support.saltfactories_compat import DaemonFactory
from tests.support.saltfactories_compat import SshdDaemonFactory as _SshdDaemonFactory
log = logging.getLogger(__name__)
USERNAME = "gitpillaruser"
PASSWORD = "saltrules"
_OPTS = {
"__role": "minion",
"environment": None,
"pillarenv": None,
"hash_type": "sha256",
"file_roots": {},
"state_top": "top.sls",
"state_top_saltenv": None,
"renderer": "yaml_jinja",
"renderer_whitelist": [],
"renderer_blacklist": [],
"pillar_merge_lists": False,
"git_pillar_base": "master",
"git_pillar_branch": "master",
"git_pillar_env": "",
"git_pillar_fallback": "",
"git_pillar_root": "",
"git_pillar_ssl_verify": True,
"git_pillar_global_lock": True,
"git_pillar_user": "",
"git_pillar_password": "",
"git_pillar_insecure_auth": False,
"git_pillar_privkey": "",
"git_pillar_pubkey": "",
"git_pillar_passphrase": "",
"git_pillar_refspecs": [
"+refs/heads/*:refs/remotes/origin/*",
"+refs/tags/*:refs/tags/*",
],
"git_pillar_includes": True,
}
PROC_TIMEOUT = 10
_OPTS = freeze(
{
"__role": "minion",
"environment": None,
"pillarenv": None,
"hash_type": "sha256",
"file_roots": {},
"state_top": "top.sls",
"state_top_saltenv": None,
"renderer": "yaml_jinja",
"renderer_whitelist": [],
"renderer_blacklist": [],
"pillar_merge_lists": False,
"git_pillar_base": "master",
"git_pillar_branch": "master",
"git_pillar_env": "",
"git_pillar_fallback": "",
"git_pillar_root": "",
"git_pillar_ssl_verify": True,
"git_pillar_global_lock": True,
"git_pillar_user": "",
"git_pillar_password": "",
"git_pillar_insecure_auth": False,
"git_pillar_privkey": "",
"git_pillar_pubkey": "",
"git_pillar_passphrase": "",
"git_pillar_refspecs": [
"+refs/heads/*:refs/remotes/origin/*",
"+refs/tags/*:refs/tags/*",
],
"git_pillar_includes": True,
}
)
class UwsgiDaemon(FactoryDaemonScriptBase):
def __init__(self, *args, **kwargs):
config_dir = kwargs.pop("config_dir")
check_port = kwargs.pop("check_port")
super().__init__(*args, **kwargs)
self.config_dir = config_dir
self.check_port = check_port
class SshdDaemonFactory(_SshdDaemonFactory):
def apply_pre_start_states(self, salt_call_cli, testclass, username):
if self.listen_port in self.check_ports:
self.check_ports.remove(self.listen_port)
if self.listen_port in self.listen_ports:
self.listen_ports.remove(self.listen_port)
self.listen_port = get_unused_localhost_port()
self.check_ports.append(self.listen_port)
self.listen_ports.append(self.listen_port)
url = "ssh://{username}@127.0.0.1:{port}/~/repo.git".format(
username=testclass.username, port=self.listen_port
)
url_extra_repo = "ssh://{username}@127.0.0.1:{port}/~/extra_repo.git".format(
username=testclass.username, port=self.listen_port
)
home = "/root/.ssh"
testclass.ext_opts = {
"url": url,
"url_extra_repo": url_extra_repo,
"privkey_nopass": os.path.join(home, testclass.id_rsa_nopass),
"pubkey_nopass": os.path.join(home, testclass.id_rsa_nopass + ".pub"),
"privkey_withpass": os.path.join(home, testclass.id_rsa_withpass),
"pubkey_withpass": os.path.join(home, testclass.id_rsa_withpass + ".pub"),
"passphrase": testclass.passphrase,
}
ret = salt_call_cli.run(
"state.apply",
mods="git_pillar.ssh",
pillar={
"git_pillar": {
"git_ssh": testclass.git_ssh,
"id_rsa_nopass": testclass.id_rsa_nopass,
"id_rsa_withpass": testclass.id_rsa_withpass,
"sshd_bin": self.get_script_path(),
"sshd_port": self.listen_port,
"sshd_config_dir": str(self.config_dir),
"master_user": username,
"user": testclass.username,
}
},
_timeout=240,
)
if ret.exitcode != 0:
pytest.fail("Failed to apply the 'git_pillar.ssh' state")
if next(iter(ret.json.values()))["result"] is not True:
pytest.fail("Failed to apply the 'git_pillar.ssh' state")
def get_log_prefix(self):
return "[uWSGI] "
def set_known_host(self, salt_call_cli, username):
ret = salt_call_cli.run(
"ssh.set_known_host",
user=username,
hostname="127.0.0.1",
port=self.listen_port,
enc="ssh-rsa",
fingerprint="fd:6f:7f:5d:06:6b:f2:06:0d:26:93:9e:5a:b5:19:46",
hash_known_hosts=False,
fingerprint_hash_type="md5",
)
if ret.exitcode != 0:
pytest.fail("Failed to run 'ssh.set_known_host'")
if "error" in ret.json:
pytest.fail("Failed to run 'ssh.set_known_host'")
@attr.s(kw_only=True, slots=True)
class UwsgiDaemon(DaemonFactory):
config_dir = attr.ib()
listen_port = attr.ib(default=attr.Factory(get_unused_localhost_port))
def __attrs_post_init__(self):
if self.check_ports is None:
self.check_ports = []
self.check_ports.append(self.listen_port)
super().__attrs_post_init__()
def get_base_script_args(self):
"""
@ -95,259 +153,28 @@ class UwsgiDaemon(FactoryDaemonScriptBase):
"""
return ["--yaml", os.path.join(self.config_dir, "uwsgi.yml")]
def get_check_ports(self):
"""
Return a list of ports to check against to ensure the daemon is running
"""
return [self.check_port]
def apply_pre_start_states(self, salt_call_cli, testclass, root_dir):
if self.listen_port in self.check_ports:
self.check_ports.remove(self.listen_port)
if self.listen_port in self.listen_ports:
self.listen_ports.remove(self.listen_port)
self.listen_port = get_unused_localhost_port()
self.check_ports.append(self.listen_port)
self.listen_ports.append(self.listen_port)
config_dir = os.path.join(root_dir, "config")
git_dir = os.path.join(root_dir, "git")
testclass.repo_dir = repo_dir = os.path.join(git_dir, "repos")
venv_dir = os.path.join(root_dir, "venv")
uwsgi_bin = os.path.join(venv_dir, "bin", "uwsgi")
class NginxDaemon(FactoryDaemonScriptBase):
def __init__(self, *args, **kwargs):
config_dir = kwargs.pop("config_dir")
check_port = kwargs.pop("check_port")
super().__init__(*args, **kwargs)
self.config_dir = config_dir
self.check_port = check_port
def get_log_prefix(self):
return "[Nginx] "
def get_base_script_args(self):
"""
Returns any additional arguments to pass to the CLI script
"""
return ["-c", os.path.join(self.config_dir, "nginx.conf")]
def get_check_ports(self):
"""
Return a list of ports to check against to ensure the daemon is running
"""
return [self.check_port]
class SaltClientMixin(ModuleCase):
client = None
@classmethod
@requires_system_grains
def setUpClass(cls, grains=None): # pylint: disable=arguments-differ
# Cent OS 6 has too old a version of git to handle the make_repo code, as
# it lacks the -c option for git itself.
make_repo = getattr(cls, "make_repo", None)
if (
callable(make_repo)
and grains["os_family"] == "RedHat"
and grains["osmajorrelease"] < 7
):
raise SkipTest("RHEL < 7 has too old a version of git to run these tests")
# Late import
import salt.client
mopts = AdaptedConfigurationTestCaseMixin.get_config(
"master", from_scratch=True
)
cls.user = mopts["user"]
cls.client = salt.client.get_local_client(mopts=mopts)
@classmethod
def tearDownClass(cls):
cls.client = None
@classmethod
def cls_run_function(cls, function, *args, **kwargs):
orig = cls.client.cmd("minion", function, arg=args, timeout=300, kwarg=kwargs)
return orig["minion"]
class SSHDMixin(SaltClientMixin, SaltReturnAssertsMixin):
"""
Functions to stand up an SSHD server to serve up git repos for tests.
"""
sshd_proc = None
prep_states_ran = False
known_hosts_setup = False
@classmethod
def setUpClass(cls): # pylint: disable=arguments-differ
super().setUpClass()
try:
log.info("%s: prep_server()", cls.__name__)
cls.sshd_bin = salt.utils.path.which("sshd")
cls.sshd_config_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
cls.sshd_config = os.path.join(cls.sshd_config_dir, "sshd_config")
cls.sshd_port = get_unused_localhost_port(cached_seconds=120)
cls.url = "ssh://{username}@127.0.0.1:{port}/~/repo.git".format(
username=cls.username, port=cls.sshd_port
)
cls.url_extra_repo = "ssh://{username}@127.0.0.1:{port}/~/extra_repo.git".format(
username=cls.username, port=cls.sshd_port
)
home = "/root/.ssh"
cls.ext_opts = {
"url": cls.url,
"url_extra_repo": cls.url_extra_repo,
"privkey_nopass": os.path.join(home, cls.id_rsa_nopass),
"pubkey_nopass": os.path.join(home, cls.id_rsa_nopass + ".pub"),
"privkey_withpass": os.path.join(home, cls.id_rsa_withpass),
"pubkey_withpass": os.path.join(home, cls.id_rsa_withpass + ".pub"),
"passphrase": cls.passphrase,
}
if cls.prep_states_ran is False:
ret = cls.cls_run_function(
"state.apply",
mods="git_pillar.ssh",
pillar={
"git_pillar": {
"git_ssh": cls.git_ssh,
"id_rsa_nopass": cls.id_rsa_nopass,
"id_rsa_withpass": cls.id_rsa_withpass,
"sshd_bin": cls.sshd_bin,
"sshd_port": cls.sshd_port,
"sshd_config_dir": cls.sshd_config_dir,
"master_user": cls.user,
"user": cls.username,
}
},
)
assert next(iter(ret.values()))["result"] is True
cls.prep_states_ran = True
log.info("%s: States applied", cls.__name__)
if cls.sshd_proc is not None:
if not psutil.pid_exists(cls.sshd_proc.pid):
log.info(
"%s: sshd started but appears to be dead now. Will try to restart it.",
cls.__name__,
)
cls.sshd_proc = None
if cls.sshd_proc is None:
cls.sshd_proc = start_daemon(
cls.sshd_bin,
SshdDaemon,
config_dir=pathlib.Path(cls.sshd_config_dir),
serve_port=cls.sshd_port,
)
log.info("%s: sshd started", cls.__name__)
except AssertionError:
cls.tearDownClass()
raise
if cls.known_hosts_setup is False:
known_hosts_ret = cls.cls_run_function(
"ssh.set_known_host",
user=cls.user,
hostname="127.0.0.1",
port=cls.sshd_port,
enc="ssh-rsa",
fingerprint="fd:6f:7f:5d:06:6b:f2:06:0d:26:93:9e:5a:b5:19:46",
hash_known_hosts=False,
fingerprint_hash_type="md5",
)
if "error" in known_hosts_ret:
cls.tearDownClass()
raise AssertionError(
"Failed to add key to {} user's known_hosts "
"file: {}".format(cls.master_opts["user"], known_hosts_ret["error"])
)
cls.known_hosts_setup = True
@classmethod
def tearDownClass(cls):
if cls.sshd_proc is not None:
log.info(
"[%s] Stopping %s",
cls.sshd_proc.get_log_prefix(),
cls.sshd_proc.__class__.__name__,
)
terminate_process(cls.sshd_proc.pid, kill_children=True, slow_stop=True)
log.info(
"[%s] %s stopped",
cls.sshd_proc.get_log_prefix(),
cls.sshd_proc.__class__.__name__,
)
cls.sshd_proc = None
if cls.prep_states_ran:
ret = cls.cls_run_function(
"state.single", "user.absent", name=cls.username, purge=True
)
try:
if ret and "minion" in ret:
ret_data = next(iter(ret["minion"].values()))
if not ret_data["result"]:
log.warning("Failed to delete test account %s", cls.username)
except KeyError:
log.warning(
"Failed to delete test account. Salt return:\n%s",
pprint.pformat(ret),
)
cls.prep_states_ran = False
cls.known_hosts_setup = False
shutil.rmtree(cls.sshd_config_dir, ignore_errors=True)
ssh_dir = os.path.expanduser("~/.ssh")
for filename in (
cls.id_rsa_nopass,
cls.id_rsa_nopass + ".pub",
cls.id_rsa_withpass,
cls.id_rsa_withpass + ".pub",
cls.git_ssh,
):
try:
os.remove(os.path.join(ssh_dir, filename))
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
super().tearDownClass()
class WebserverMixin(SaltClientMixin, SaltReturnAssertsMixin):
"""
Functions to stand up an nginx + uWSGI + git-http-backend webserver to
serve up git repos for tests.
"""
nginx_proc = uwsgi_proc = None
prep_states_ran = False
@classmethod
def setUpClass(cls): # pylint: disable=arguments-differ
"""
Set up all the webserver paths. Designed to be run once in a
setUpClass function.
"""
super().setUpClass()
cls.root_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
cls.config_dir = os.path.join(cls.root_dir, "config")
cls.nginx_conf = os.path.join(cls.config_dir, "nginx.conf")
cls.uwsgi_conf = os.path.join(cls.config_dir, "uwsgi.yml")
cls.git_dir = os.path.join(cls.root_dir, "git")
cls.repo_dir = os.path.join(cls.git_dir, "repos")
cls.venv_dir = os.path.join(cls.root_dir, "venv")
cls.uwsgi_bin = os.path.join(cls.venv_dir, "bin", "uwsgi")
cls.nginx_port = cls.uwsgi_port = get_unused_localhost_port(cached_seconds=120)
cls.uwsgi_port = get_unused_localhost_port(cached_seconds=120)
cls.url = "http://127.0.0.1:{port}/repo.git".format(port=cls.nginx_port)
cls.url_extra_repo = "http://127.0.0.1:{port}/extra_repo.git".format(
port=cls.nginx_port
)
cls.ext_opts = {"url": cls.url, "url_extra_repo": cls.url_extra_repo}
# Add auth params if present (if so this will trigger the spawned
# server to turn on HTTP basic auth).
for credential_param in ("user", "password"):
if hasattr(cls, credential_param):
cls.ext_opts[credential_param] = getattr(cls, credential_param)
auth_enabled = hasattr(cls, "username") and hasattr(cls, "password")
pillar = {
"git_pillar": {
"config_dir": cls.config_dir,
"git_dir": cls.git_dir,
"venv_dir": cls.venv_dir,
"root_dir": cls.root_dir,
"nginx_port": cls.nginx_port,
"uwsgi_port": cls.uwsgi_port,
"auth_enabled": auth_enabled,
"config_dir": config_dir,
"git_dir": git_dir,
"venv_dir": venv_dir,
"root_dir": root_dir,
"uwsgi_port": self.listen_port,
}
}
@ -360,8 +187,7 @@ class WebserverMixin(SaltClientMixin, SaltReturnAssertsMixin):
git_core = "/usr/lib/git-core"
if not os.path.exists(git_core):
cls.tearDownClass()
raise AssertionError(
pytest.fail(
"{} not found. Either git is not installed, or the test "
"class needs to be updated.".format(git_core)
)
@ -369,90 +195,217 @@ class WebserverMixin(SaltClientMixin, SaltReturnAssertsMixin):
pillar["git_pillar"]["git-http-backend"] = os.path.join(
git_core, "git-http-backend"
)
try:
if cls.prep_states_ran is False:
ret = cls.cls_run_function(
"state.apply", mods="git_pillar.http", pillar=pillar
)
assert next(iter(ret.values()))["result"] is True
cls.prep_states_ran = True
log.info("%s: States applied", cls.__name__)
if cls.uwsgi_proc is not None:
if not psutil.pid_exists(cls.uwsgi_proc.pid):
log.warning(
"%s: uWsgi started but appears to be dead now. Will try to restart it.",
cls.__name__,
)
cls.uwsgi_proc = None
if cls.uwsgi_proc is None:
cls.uwsgi_proc = start_daemon(
cls.uwsgi_bin,
UwsgiDaemon,
config_dir=cls.config_dir,
check_port=cls.uwsgi_port,
)
log.info("%s: %s started", cls.__name__, cls.uwsgi_bin)
if cls.nginx_proc is not None:
if not psutil.pid_exists(cls.nginx_proc.pid):
log.warning(
"%s: nginx started but appears to be dead now. Will try to restart it.",
cls.__name__,
)
cls.nginx_proc = None
if cls.nginx_proc is None:
cls.nginx_proc = start_daemon(
"nginx",
NginxDaemon,
config_dir=cls.config_dir,
check_port=cls.nginx_port,
)
log.info("%s: nginx started", cls.__name__)
except AssertionError:
cls.tearDownClass()
raise
@classmethod
def tearDownClass(cls):
if cls.nginx_proc is not None:
log.info(
"[%s] Stopping %s",
cls.nginx_proc.get_log_prefix(),
cls.nginx_proc.__class__.__name__,
)
terminate_process(cls.nginx_proc.pid, kill_children=True, slow_stop=True)
log.info(
"[%s] %s stopped",
cls.nginx_proc.get_log_prefix(),
cls.nginx_proc.__class__.__name__,
)
cls.nginx_proc = None
if cls.uwsgi_proc is not None:
log.info(
"[%s] Stopping %s",
cls.uwsgi_proc.get_log_prefix(),
cls.uwsgi_proc.__class__.__name__,
)
terminate_process(cls.uwsgi_proc.pid, kill_children=True, slow_stop=True)
log.info(
"[%s] %s stopped",
cls.uwsgi_proc.get_log_prefix(),
cls.uwsgi_proc.__class__.__name__,
)
cls.uwsgi_proc = None
shutil.rmtree(cls.root_dir, ignore_errors=True)
cls.prep_states_ran = False
super().tearDownClass()
ret = salt_call_cli.run(
"state.apply", mods="git_pillar.http.uwsgi", pillar=pillar, _timeout=120
)
if ret.exitcode != 0:
pytest.fail("Failed to apply the 'git_pillar.http.uwsgi' state")
if next(iter(ret.json.values()))["result"] is not True:
pytest.fail("Failed to apply the 'git_pillar.http.uwsgi' state")
@attr.s(kw_only=True, slots=True)
class NginxDaemon(DaemonFactory):
config_dir = attr.ib()
uwsgi_port = attr.ib()
listen_port = attr.ib(default=attr.Factory(get_unused_localhost_port))
def __attrs_post_init__(self):
if self.check_ports is None:
self.check_ports = []
self.check_ports.append(self.listen_port)
super().__attrs_post_init__()
def get_base_script_args(self):
"""
Returns any additional arguments to pass to the CLI script
"""
return ["-c", os.path.join(self.config_dir, "nginx.conf")]
def apply_pre_start_states(self, salt_call_cli, testclass, root_dir):
if self.listen_port in self.check_ports:
self.check_ports.remove(self.listen_port)
if self.listen_port in self.listen_ports:
self.listen_ports.remove(self.listen_port)
self.listen_port = get_unused_localhost_port()
self.check_ports.append(self.listen_port)
self.listen_ports.append(self.listen_port)
config_dir = os.path.join(root_dir, "config")
git_dir = os.path.join(root_dir, "git")
url = "http://127.0.0.1:{port}/repo.git".format(port=self.listen_port)
url_extra_repo = "http://127.0.0.1:{port}/extra_repo.git".format(
port=self.listen_port
)
ext_opts = {"url": url, "url_extra_repo": url_extra_repo}
# Add auth params if present (if so this will trigger the spawned
# server to turn on HTTP basic auth).
for credential_param in ("user", "password"):
if hasattr(testclass, credential_param):
ext_opts[credential_param] = getattr(testclass, credential_param)
testclass.ext_opts = ext_opts
testclass.nginx_port = self.listen_port
auth_enabled = hasattr(testclass, "username") and hasattr(testclass, "password")
pillar = {
"git_pillar": {
"config_dir": config_dir,
"git_dir": git_dir,
"uwsgi_port": self.uwsgi_port,
"nginx_port": self.listen_port,
"auth_enabled": auth_enabled,
}
}
ret = salt_call_cli.run(
"state.apply", mods="git_pillar.http.nginx", pillar=pillar
)
if ret.exitcode != 0:
pytest.fail("Failed to apply the 'git_pillar.http.nginx' state")
if next(iter(ret.json.values()))["result"] is not True:
pytest.fail("Failed to apply the 'git_pillar.http.nginx' state")
@pytest.fixture(scope="class")
def ssh_pillar_tests_prep(request, salt_factories, salt_master, salt_minion):
"""
Stand up an SSHD server to serve up git repos for tests.
"""
try:
salt_call_cli = salt_minion.get_salt_call_cli()
raise RuntimeError("s0undt3ch, it's time to cleanup this spaghetti code!")
except AttributeError:
salt_call_cli = salt_factories.get_salt_call_cli(salt_minion.config["id"])
sshd_bin = salt.utils.path.which("sshd")
sshd_config_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
sshd_proc = SshdDaemonFactory(
cli_script_name=sshd_bin,
config_dir=sshd_config_dir,
start_timeout=120,
display_name=request.cls.__name__,
)
sshd_proc.register_before_start_callback(
sshd_proc.apply_pre_start_states,
salt_call_cli=salt_call_cli,
testclass=request.cls,
username=salt_master.config["user"],
)
sshd_proc.register_after_start_callback(
sshd_proc.set_known_host,
salt_call_cli=salt_call_cli,
username=salt_master.config["user"],
)
try:
sshd_proc.start()
yield
finally:
request.cls.ext_opts = None
salt_call_cli.run(
"state.single", "user.absent", name=request.cls.username, purge=True
)
shutil.rmtree(sshd_config_dir, ignore_errors=True)
ssh_dir = os.path.expanduser("~/.ssh")
for filename in (
request.cls.id_rsa_nopass,
request.cls.id_rsa_nopass + ".pub",
request.cls.id_rsa_withpass,
request.cls.id_rsa_withpass + ".pub",
request.cls.git_ssh,
):
try:
os.remove(os.path.join(ssh_dir, filename))
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
sshd_proc.terminate()
@pytest.fixture(scope="class")
def webserver_pillar_tests_prep(request, salt_factories, salt_master, salt_minion):
"""
Stand up an nginx + uWSGI + git-http-backend webserver to
serve up git repos for tests.
"""
try:
salt_call_cli = salt_minion.get_salt_call_cli()
raise RuntimeError("s0undt3ch, it's time to cleanup this spaghetti code!")
except AttributeError:
salt_call_cli = salt_factories.get_salt_call_cli(salt_minion.config["id"])
root_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
config_dir = os.path.join(root_dir, "config")
venv_dir = os.path.join(root_dir, "venv")
uwsgi_bin = os.path.join(venv_dir, "bin", "uwsgi")
uwsgi_proc = nginx_proc = None
try:
uwsgi_proc = UwsgiDaemon(
cli_script_name=uwsgi_bin,
config_dir=config_dir,
start_timeout=120,
display_name=request.cls.__name__,
)
uwsgi_proc.register_before_start_callback(
uwsgi_proc.apply_pre_start_states,
salt_call_cli=salt_call_cli,
testclass=request.cls,
root_dir=root_dir,
)
uwsgi_proc.start()
nginx_proc = NginxDaemon(
cli_script_name="nginx",
config_dir=config_dir,
start_timeout=120,
uwsgi_port=uwsgi_proc.listen_port,
display_name=request.cls.__name__,
)
nginx_proc.register_before_start_callback(
nginx_proc.apply_pre_start_states,
salt_call_cli=salt_call_cli,
testclass=request.cls,
root_dir=root_dir,
)
nginx_proc.start()
yield
finally:
request.cls.repo_dir = request.cls.ext_opts = request.cls.nginx_port = None
if uwsgi_proc:
uwsgi_proc.terminate()
if nginx_proc:
nginx_proc.terminate()
shutil.rmtree(root_dir, ignore_errors=True)
@pytest.fixture(scope="class")
def webserver_pillar_tests_prep_authenticated(request, webserver_pillar_tests_prep):
url = "http://{username}:{password}@127.0.0.1:{port}/repo.git".format(
username=request.cls.username,
password=request.cls.password,
port=request.cls.nginx_port,
)
url_extra_repo = "http://{username}:{password}@127.0.0.1:{port}/extra_repo.git".format(
username=request.cls.username,
password=request.cls.password,
port=request.cls.nginx_port,
)
request.cls.ext_opts["url"] = url
request.cls.ext_opts["url_extra_repo"] = url_extra_repo
request.cls.ext_opts["username"] = request.cls.username
request.cls.ext_opts["password"] = request.cls.password
@SKIP_IF_NOT_RUNNING_PYTEST
class GitTestBase(ModuleCase):
"""
Base class for all gitfs/git_pillar tests. Must be subclassed and paired
with either SSHDMixin or WebserverMixin to provide the server.
Base class for all gitfs/git_pillar tests.
"""
maxDiff = None
git_opts = '-c user.name="Foo Bar" -c user.email=foo@bar.com'
ext_opts = {}
def make_repo(self, root_dir, user="root"):
raise NotImplementedError()
@ -465,7 +418,7 @@ class GitFSTestBase(GitTestBase, LoaderModuleMockMixin):
@requires_system_grains
def setup_loader_modules(self, grains): # pylint: disable=W0221
return {gitfs: {"__opts__": copy.copy(_OPTS), "__grains__": grains}}
return {gitfs: {"__opts__": _OPTS.copy(), "__grains__": grains}}
def make_repo(self, root_dir, user="root"):
raise NotImplementedError()
@ -481,7 +434,7 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin):
@requires_system_grains
def setup_loader_modules(self, grains): # pylint: disable=W0221
return {git_pillar: {"__opts__": copy.copy(_OPTS), "__grains__": grains}}
return {git_pillar: {"__opts__": _OPTS.copy(), "__grains__": grains}}
def get_pillar(self, ext_pillar_conf):
"""
@ -752,7 +705,7 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin):
shutil.rmtree(dirname, ignore_errors=True)
class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin):
class GitPillarSSHTestBase(GitPillarTestBase):
"""
Base class for GitPython and Pygit2 SSH tests
"""
@ -786,7 +739,7 @@ class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin):
return super().get_pillar(ext_pillar_conf)
class GitPillarHTTPTestBase(GitPillarTestBase, WebserverMixin):
class GitPillarHTTPTestBase(GitPillarTestBase):
"""
Base class for GitPython and Pygit2 HTTP tests
"""

9
tests/unit/runners/test_git_pillar.py
View file

@ -1,22 +1,15 @@
# -*- coding: utf-8 -*-
"""
unit tests for the git_pillar runner
"""
# Import Python Libs
from __future__ import absolute_import, print_function, unicode_literals
import copy
import errno
import logging
import tempfile
# Import Salt Libs
import salt.runners.git_pillar as git_pillar
import salt.utils.files
import salt.utils.gitfs
# Import Salt Testing Libs
from tests.support.gitfs import _OPTS
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.mock import patch
@ -46,7 +39,7 @@ class GitPillarTest(TestCase, LoaderModuleMockMixin):
raise
def setup_loader_modules(self):
opts = copy.copy(_OPTS)
opts = _OPTS.copy()
opts["cachedir"] = self.tmp_cachedir
opts["verified_git_pillar_provider"] = "gitfoo"
opts["ext_pillar"] = [