saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #58293 from s0undt3ch/hotfix/pytest-migration-1

Browse source 
[PyTest Migration] Minor fixes & Improvements to PyTest runs
This commit is contained in:
Pedro Algarvio 2020-08-26 20:38:54 +01:00 • committed by GitHub
commit 252a934448
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 122 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
noxfile.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
"""
noxfile
~~~~~~~
@ -7,7 +6,6 @@ Nox configuration script
"""
# pylint: disable=resource-leakage,3rd-party-module-not-gated
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import glob
@ -65,7 +63,7 @@ RUNTESTS_LOGFILE = os.path.join(
)
# Prevent Python from writing bytecode
os.environ[str("PYTHONDONTWRITEBYTECODE")] = str("1")
os.environ["PYTHONDONTWRITEBYTECODE"] = "1"
def find_session_runner(session, name, **kwargs):
@ -952,7 +950,15 @@ def _pytest(session, coverage, cmd_args):
cmd_args.append("--lf")
if coverage is True:
_run_with_coverage(
session, "python", "-m", "coverage", "run", "-m", "pytest", *cmd_args
session,
"python",
"-m",
"coverage",
"run",
"-m",
"pytest",
"--showlocals",
*cmd_args
)
else:
session.run("python", "-m", "pytest", *cmd_args, env=env)

59
tests/pytests/unit/utils/test_msgpack.py
View file

@ -1,5 +1,3 @@
import os
import pytest
import salt.utils.msgpack
from tests.support.mock import MagicMock, patch
@ -14,17 +12,16 @@ def test_load_encoding(tmpdir):
https://github.com/msgpack/msgpack-python/blob/master/ChangeLog.rst
"""
fn_ = os.path.join(tmpdir, "test_load_encoding.txt")
salt.utils.msgpack.version = (1, 0, 0)
fname = tmpdir.join("test_load_encoding.txt")
kwargs = {"encoding": "utf-8"}
data = [1, 2, 3]
with salt.utils.files.fopen(fn_, "wb") as fh_:
salt.utils.msgpack.dump(data, fh_)
with salt.utils.files.fopen(fn_, "rb") as fh_:
ret = salt.utils.msgpack.load(fh_, **kwargs)
with patch.object(salt.utils.msgpack, "version", (1, 0, 0)):
with salt.utils.files.fopen(fname.strpath, "wb") as wfh:
salt.utils.msgpack.dump(data, wfh)
with salt.utils.files.fopen(fname.strpath, "rb") as rfh:
ret = salt.utils.msgpack.load(rfh, **kwargs)
assert ret == data
assert ret == data
@pytest.mark.parametrize(
@ -35,28 +32,28 @@ def test_load_multiple_versions(version, encoding, tmpdir):
test when using msgpack on multiple versions that
we only remove encoding on >= 1.0.0
"""
fn_ = os.path.join(tmpdir, "test_load_multipl_versions.txt")
salt.utils.msgpack.version = version
data = [1, 2, 3]
fname = tmpdir.join("test_load_multipl_versions.txt")
with patch.object(salt.utils.msgpack, "version", version):
data = [1, 2, 3]
mock_dump = MagicMock(return_value=data)
patch_dump = patch("msgpack.pack", mock_dump)
mock_dump = MagicMock(return_value=data)
patch_dump = patch("msgpack.pack", mock_dump)
mock_load = MagicMock(return_value=data)
patch_load = patch("msgpack.unpack", mock_load)
mock_load = MagicMock(return_value=data)
patch_load = patch("msgpack.unpack", mock_load)
kwargs = {"encoding": "utf-8"}
with patch_dump, patch_load:
with salt.utils.files.fopen(fn_, "wb") as fh_:
salt.utils.msgpack.dump(data, fh_, encoding="utf-8")
if encoding:
assert "encoding" in mock_dump.call_args.kwargs
else:
assert "encoding" not in mock_dump.call_args.kwargs
kwargs = {"encoding": "utf-8"}
with patch_dump, patch_load:
with salt.utils.files.fopen(fname.strpath, "wb") as wfh:
salt.utils.msgpack.dump(data, wfh, encoding="utf-8")
if encoding:
assert "encoding" in mock_dump.call_args.kwargs
else:
assert "encoding" not in mock_dump.call_args.kwargs
with salt.utils.files.fopen(fn_, "rb") as fh_:
salt.utils.msgpack.load(fh_, **kwargs)
if encoding:
assert "encoding" in mock_load.call_args.kwargs
else:
assert "encoding" not in mock_load.call_args.kwargs
with salt.utils.files.fopen(fname.strpath, "rb") as rfh:
salt.utils.msgpack.load(rfh, **kwargs)
if encoding:
assert "encoding" in mock_load.call_args.kwargs
else:
assert "encoding" not in mock_load.call_args.kwargs

35
tests/support/helpers.py
View file

@ -1739,3 +1739,38 @@ def change_cwd(path):
finally:
# Restore Old CWD
os.chdir(old_cwd)
@functools.lru_cache(maxsize=1)
def get_virtualenv_binary_path():
# Under windows we can't seem to properly create a virtualenv off of another
# virtualenv, we can on linux but we will still point to the virtualenv binary
# outside the virtualenv running the test suite, if that's the case.
try:
real_prefix = sys.real_prefix
# The above attribute exists, this is a virtualenv
if salt.utils.platform.is_windows():
virtualenv_binary = os.path.join(real_prefix, "Scripts", "virtualenv.exe")
else:
# We need to remove the virtualenv from PATH or we'll get the virtualenv binary
# from within the virtualenv, we don't want that
path = os.environ.get("PATH")
if path is not None:
path_items = path.split(os.pathsep)
for item in path_items[:]:
if item.startswith(sys.base_prefix):
path_items.remove(item)
os.environ["PATH"] = os.pathsep.join(path_items)
virtualenv_binary = salt.utils.path.which("virtualenv")
if path is not None:
# Restore previous environ PATH
os.environ["PATH"] = path
if not virtualenv_binary.startswith(real_prefix):
virtualenv_binary = None
if virtualenv_binary and not os.path.exists(virtualenv_binary):
# It doesn't exist?!
virtualenv_binary = None
except AttributeError:
# We're not running inside a virtualenv
virtualenv_binary = None
return virtualenv_binary

46
tests/support/pytest/fixtures.py
View file

@ -11,55 +11,15 @@
import os
import shutil
import stat
import sys
import pytest
import salt.utils.files
from salt.serializers import yaml
from salt.utils.immutabletypes import freeze
from tests.support.helpers import get_virtualenv_binary_path
from tests.support.runtests import RUNTIME_VARS
def _get_virtualenv_binary_path():
try:
return _get_virtualenv_binary_path.__virtualenv_binary__
except AttributeError:
# Under windows we can't seem to properly create a virtualenv off of another
# virtualenv, we can on linux but we will still point to the virtualenv binary
# outside the virtualenv running the test suite, if that's the case.
try:
real_prefix = sys.real_prefix
# The above attribute exists, this is a virtualenv
if salt.utils.platform.is_windows():
virtualenv_binary = os.path.join(
real_prefix, "Scripts", "virtualenv.exe"
)
else:
# We need to remove the virtualenv from PATH or we'll get the virtualenv binary
# from within the virtualenv, we don't want that
path = os.environ.get("PATH")
if path is not None:
path_items = path.split(os.pathsep)
for item in path_items[:]:
if item.startswith(sys.base_prefix):
path_items.remove(item)
os.environ["PATH"] = os.pathsep.join(path_items)
virtualenv_binary = salt.utils.path.which("virtualenv")
if path is not None:
# Restore previous environ PATH
os.environ["PATH"] = path
if not virtualenv_binary.startswith(real_prefix):
virtualenv_binary = None
if virtualenv_binary and not os.path.exists(virtualenv_binary):
# It doesn't exist?!
virtualenv_binary = None
except AttributeError:
# We're not running inside a virtualenv
virtualenv_binary = None
_get_virtualenv_binary_path.__virtualenv_binary__ = virtualenv_binary
return virtualenv_binary
@pytest.fixture(scope="session")
def integration_files_dir(salt_factories):
"""
@ -366,7 +326,7 @@ def salt_minion_config(request, salt_factories, salt_master_config):
"prod": [RUNTIME_VARS.TMP_PRODENV_PILLAR_TREE],
},
}
virtualenv_binary = _get_virtualenv_binary_path()
virtualenv_binary = get_virtualenv_binary_path()
if virtualenv_binary:
config_overrides["venv_bin"] = virtualenv_binary
return salt_factories.configure_minion(
@ -408,7 +368,7 @@ def salt_sub_minion_config(request, salt_factories, salt_master_config):
"prod": [RUNTIME_VARS.TMP_PRODENV_PILLAR_TREE],
},
}
virtualenv_binary = _get_virtualenv_binary_path()
virtualenv_binary = get_virtualenv_binary_path()
if virtualenv_binary:
config_overrides["venv_bin"] = virtualenv_binary
return salt_factories.configure_minion(

71
tests/support/saltfactories_compat.py
View file

@ -452,7 +452,7 @@ class DaemonFactory(SubprocessFactoryBase):
def _format_callback(self, callback, args, kwargs):
callback_str = "{}(".format(callback.__name__)
if args:
callback_str += ", ".join(args)
callback_str += ", ".join([repr(arg) for arg in args])
if kwargs:
callback_str += ", ".join(
["{}={!r}".format(k, v) for (k, v) in kwargs.items()]
@ -468,16 +468,6 @@ class DaemonFactory(SubprocessFactoryBase):
log.warning("%s is already running.", self)
return True
process_running = False
for callback, args, kwargs in self.before_start_callbacks:
try:
callback(*args, **kwargs)
except Exception as exc: # pylint: disable=broad-except
log.info(
"Exception raised when running %s: %s",
self._format_callback(callback, args, kwargs),
exc,
exc_info=True,
)
start_time = time.time()
start_attempts = max_start_attempts or self.max_start_attempts
current_attempt = 0
@ -491,6 +481,16 @@ class DaemonFactory(SubprocessFactoryBase):
log.info(
"Starting %s. Attempt: %d of %d", self, current_attempt, start_attempts
)
for callback, args, kwargs in self.before_start_callbacks:
try:
callback(*args, **kwargs)
except Exception as exc: # pylint: disable=broad-except
log.info(
"Exception raised when running %s: %s",
self._format_callback(callback, args, kwargs),
exc,
exc_info=True,
)
current_start_time = time.time()
start_running_timeout = current_start_time + (
start_timeout or self.start_timeout
@ -511,12 +511,16 @@ class DaemonFactory(SubprocessFactoryBase):
log.warning("%s is no longer running", self)
self.terminate()
break
if (
self.run_start_checks(current_start_time, start_running_timeout)
is False
):
time.sleep(1)
continue
try:
if (
self.run_start_checks(current_start_time, start_running_timeout)
is False
):
time.sleep(1)
continue
except FactoryNotStarted:
self.terminate()
break
log.info(
"The %s factory is running after %d attempts. Took %1.2f seconds",
self,
@ -600,11 +604,12 @@ class DaemonFactory(SubprocessFactoryBase):
checks_start_time = time.time()
while time.time() <= timeout_at:
if not self.is_running():
log.warning("%s is no longer running", self)
return False
raise FactoryNotStarted("{} is no longer running".format(self))
if not check_ports:
break
check_ports -= ports.get_connectable_ports(check_ports)
if check_ports:
time.sleep(0.5)
else:
log.error(
"Failed to check ports after %1.2f seconds for %s",
@ -644,14 +649,18 @@ class DaemonFactory(SubprocessFactoryBase):
# we have set on listen_ports, terminate those processes.
found_processes = []
for process in psutil.process_iter(["connections"]):
for connection in process.connections():
if connection.status != psutil.CONN_LISTEN:
# We only care about listening services
continue
if connection.laddr.port in self.check_ports:
found_processes.append(process)
# We already found one connection, no need to check the others
break
try:
for connection in process.connections():
if connection.status != psutil.CONN_LISTEN:
# We only care about listening services
continue
if connection.laddr.port in self.check_ports:
found_processes.append(process)
# We already found one connection, no need to check the others
break
except psutil.AccessDenied:
# We've been denied access to this process connections. Carry on.
continue
if found_processes:
log.debug(
"The following processes were found listening on ports %s: %s",
@ -946,7 +955,6 @@ class SaltDaemonFactory(SaltFactory, DaemonFactory):
def verify_config(cls, config):
salt.utils.verify.verify_env(
cls._get_verify_config_entries(config),
# running_username(),
salt.utils.user.get_user(),
pki_dir=config.get("pki_dir") or "",
root_dir=config["root_dir"],
@ -990,7 +998,9 @@ class SaltDaemonFactory(SaltFactory, DaemonFactory):
if not super().run_start_checks(started_at, timeout_at):
return False
if not self.event_listener:
log.debug("No event listener available")
log.debug(
"The 'event_listener' attribute is not set. Not checking events..."
)
return True
check_events = set(self.get_check_events())
@ -1000,8 +1010,7 @@ class SaltDaemonFactory(SaltFactory, DaemonFactory):
checks_start_time = time.time()
while time.time() <= timeout_at:
if not self.is_running():
log.info("%s is no longer running", self)
return False
raise FactoryNotStarted("{} is no longer running".format(self))
if not check_events:
break
check_events -= self.event_listener.get_events(

21
tests/unit/modules/test_aptpkg.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
"""
:synopsis: Unit Tests for Advanced Packaging Tool module 'module.aptpkg'
:platform: Linux
@ -6,20 +5,12 @@
versionadded:: 2017.7.0
"""
# Import Python Libs
from __future__ import absolute_import, print_function, unicode_literals
import copy
import logging
import textwrap
import salt.modules.aptpkg as aptpkg
from salt.exceptions import CommandExecutionError, SaltInvocationError
# Import Salt Libs
from salt.ext import six
# Import Salt Testing Libs
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.mock import MagicMock, Mock, call, patch
from tests.support.unit import TestCase, skipIf
@ -153,8 +144,8 @@ Reading state information...
0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
"""
UNINSTALL = {"tmux": {"new": six.text_type(), "old": "1.8-5"}}
INSTALL = {"tmux": {"new": "1.8-5", "old": six.text_type()}}
UNINSTALL = {"tmux": {"new": "", "old": "1.8-5"}}
INSTALL = {"tmux": {"new": "1.8-5", "old": ""}}
def _get_uri(repo):
@ -167,7 +158,7 @@ def _get_uri(repo):
return val
class MockSourceEntry(object):
class MockSourceEntry:
def __init__(self, uri, source_type, line, invalid, file=None):
self.uri = uri
self.type = source_type
@ -181,7 +172,7 @@ class MockSourceEntry(object):
return line.split()
class MockSourceList(object):
class MockSourceList:
def __init__(self):
self.list = []
@ -870,8 +861,8 @@ class AptUtilsTestCase(TestCase, LoaderModuleMockMixin):
username="Darth Vader",
) # pylint: disable=W0106
# We should sleep 4 times
self.assertEqual(sleep_mock.call_count, 4)
# We should have sleept at least 4 times
assert sleep_mock.call_count >= 4
# We should attempt to call the cmd 5 times
self.assertEqual(cmd_mock.call_count, 5)