Merge pull request #33987 from isbm/isbm-inspectlib-cleanup

inspectlib cleanup

salt/modules/inspectlib/__init__.py

@@ -13,3 +13,35 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import os
+from salt.modules.inspectlib.exceptions import InspectorSnapshotException
+from salt.modules.inspectlib.dbhandle import DBHandle
+
+
+class EnvLoader(object):
+    '''
+    Load environment.
+    '''
+    PID_FILE = '_minion_collector.pid'
+    DB_FILE = '_minion_collector.db'
+    DEFAULT_PID_PATH = '/var/run'
+    DEFAULT_CACHE_PATH = '/var/cache/salt'
+
+    def __init__(self, cachedir=None, piddir=None, pidfilename=None):
+        '''
+        Constructor.
+
+        :param options:
+        :param db_path:
+        :param pid_file:
+        '''
+        if not cachedir and '__salt__' in globals():
+            cachedir = globals().get('__salt__')['config.get']('inspector.db', '')
+
+        self.dbfile = os.path.join(cachedir or self.DEFAULT_CACHE_PATH, self.DB_FILE)
+        self.db = DBHandle(self.dbfile)
+
+        if not piddir and '__salt__' in globals():
+            piddir = globals().get('__salt__')['config.get']('inspector.pid', '')
+        self.pidfile = os.path.join(piddir or self.DEFAULT_PID_PATH, pidfilename or self.PID_FILE)

salt/modules/inspectlib/collector.py

@@ -19,16 +19,26 @@ from __future__ import absolute_import, print_function
 import os
 import sys
 from subprocess import Popen, PIPE, STDOUT
+import logging
 
 # Import Salt Libs
-from salt.modules.inspectlib.dbhandle import DBHandle
 from salt.modules.inspectlib.exceptions import (InspectorSnapshotException)
+from salt.modules.inspectlib import EnvLoader
+from salt.modules.inspectlib import kiwiproc
 import salt.utils
 from salt.utils import fsutils
 from salt.utils import reinit_crypto
+from salt.exceptions import CommandExecutionError
+
+try:
+    import kiwi
+except ImportError:
+    kiwi = None
+
+log = logging.getLogger(__name__)
 
 
-class Inspector(object):
+class Inspector(EnvLoader):
     DEFAULT_MINION_CONFIG_PATH = '/etc/salt/minion'
 
     MODE = ['configuration', 'payload', 'all']
@@ -38,26 +48,14 @@ class Inspector(object):
                     "/var/lib/rpm", "/.snapshots", "/.zfs", "/etc/ssh",
                     "/root", "/home"]
 
-    def __init__(self, db_path=None, pid_file=None):
-        # Configured path
-        if not db_path and '__salt__' in globals():
-            db_path = globals().get('__salt__')['config.get']('inspector.db', '')
+    def __init__(self, cachedir=None, piddir=None, pidfilename=None):
+        EnvLoader.__init__(self, cachedir=cachedir, piddir=piddir, pidfilename=pidfilename)
 
-        if not db_path:
-            raise InspectorSnapshotException('Inspector database location is not configured yet in minion.\n'
-                                             'Add "inspector.db: /path/to/cache" in "/etc/salt/minion".')
-        self.dbfile = db_path
-
-        self.db = DBHandle(self.dbfile)
-        self.db.open()
-
-        if not pid_file and '__salt__' in globals():
-            pid_file = globals().get('__salt__')['config.get']('inspector.pid', '')
-
-        if not pid_file:
-            raise InspectorSnapshotException("Inspector PID file location is not configured yet in minion.\n"
-                                             'Add "inspector.pid: /path/to/pids in "/etc/salt/minion".')
-        self.pidfile = pid_file
+        # TODO: This is nasty. Need to do something with this better. ASAP!
+        try:
+            self.db.open()
+        except Exception as ex:
+            log.error('Unable to [re]open db. Already opened?')
 
     def _syscall(self, command, input=None, env=None, *params):
         '''
@@ -411,7 +409,34 @@ class Inspector(object):
         self._prepare_full_scan(**kwargs)
 
         os.system("nice -{0} python {1} {2} {3} {4} & > /dev/null".format(
-            priority, __file__, self.pidfile, self.dbfile, mode))
+            priority, __file__, os.path.dirname(self.pidfile), os.path.dirname(self.dbfile), mode))
+
+    def export(self, description, local=False, path='/tmp', format='qcow2'):
+        '''
+        Export description for Kiwi.
+
+        :param local:
+        :param path:
+        :return:
+        '''
+        kiwiproc.__salt__ = __salt__
+        return kiwiproc.KiwiExporter(grains=__grains__,
+                                     format=format).load(**description).export('something')
+
+    def build(self, format='qcow2', path='/tmp'):
+        '''
+        Build an image using Kiwi.
+
+        :param format:
+        :param path:
+        :return:
+        '''
+        if kiwi is None:
+            msg = 'Unable to build the image due to the missing dependencies: Kiwi module is not available.'
+            log.error(msg)
+            raise CommandExecutionError(msg)
+
+        raise CommandExecutionError("Build is not yet implemented")
 
 
 def is_alive(pidfile):
@@ -458,7 +483,7 @@ if __name__ == '__main__':
         pid = os.fork()
         if pid > 0:
             reinit_crypto()
-            fpid = open(pidfile, "w")
+            fpid = open(os.path.join(pidfile, EnvLoader.PID_FILE), "w")
             fpid.write("{0}\n".format(pid))
             fpid.close()
             sys.exit(0)

salt/modules/inspectlib/exceptions.py

@@ -31,3 +31,9 @@ class SIException(Exception):
     '''
     System information exception.
     '''
+
+
+class InspectorKiwiProcessorException(Exception):
+    '''
+    Kiwi builder/exporter exception.
+    '''

salt/modules/inspectlib/kiwiproc.py

@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2016 SUSE LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import grp
+import pwd
+from lxml import etree
+from xml.dom import minidom
+import platform
+import socket
+from salt.modules.inspectlib.exceptions import InspectorKiwiProcessorException
+
+
+class KiwiExporter(object):
+    '''
+    Exports system description as Kiwi configuration.
+    '''
+    def __init__(self, grains, format):
+        self.__grains__ = grains
+        self.format = format
+        self._data = type('data', (), {})
+        self.name = None
+
+    def load(self, **descr):
+        '''
+        Load data by keys.
+
+        :param data:
+        :return:
+        '''
+        for obj, data in descr.items():
+            setattr(self._data, obj, data)
+
+        return self
+
+    def export(self, name):
+        '''
+        Export to the Kiwi config.xml as text.
+
+        :return:
+        '''
+
+        self.name = name
+        root = self._create_doc()
+        self._set_description(root)
+        self._set_preferences(root)
+        self._set_repositories(root)
+        self._set_users(root)
+        self._set_packages(root)
+
+        return '\n'.join([line for line in minidom.parseString(
+            etree.tostring(root, encoding='UTF-8', pretty_print=True)).toprettyxml(indent="  ").split("\n")
+                          if line.strip()])
+
+    def _get_package_manager(self):
+        '''
+        Get package manager.
+
+        :return:
+        '''
+        ret = None
+        if self.__grains__.get('os_family') in ('Kali', 'Debian'):
+            ret = 'apt-get'
+        elif self.__grains__.get('os_family', '') == 'Suse':
+            ret = 'zypper'
+        elif self.__grains__.get('os_family', '') == 'redhat':
+            ret = 'yum'
+
+        if ret is None:
+            raise InspectorKiwiProcessorException('Unsupported platform: {0}'.format(self.__grains__.get('os_family')))
+
+        return ret
+
+    def _set_preferences(self, node):
+        '''
+        Set preferences.
+
+        :return:
+        '''
+        pref = etree.SubElement(node, 'preferences')
+        pacman = etree.SubElement(pref, 'packagemanager')
+        pacman.text = self._get_package_manager()
+        p_version = etree.SubElement(pref, 'version')
+        p_version.text = '0.0.1'
+        p_type = etree.SubElement(pref, 'type')
+        p_type.set('image', 'vmx')
+
+        for disk_id, disk_data in self._data.system.get('disks', {}).items():
+            if disk_id.startswith('/dev'):
+                p_type.set('filesystem', disk_data.get('type') or 'ext3')
+                break
+
+        p_type.set('installiso', 'true')
+        p_type.set('boot', "vmxboot/suse-leap42.1")
+        p_type.set('format', self.format)
+        p_type.set('bootloader', 'grub2')
+        p_type.set('timezone', __salt__['timezone.get_zone']())
+        p_type.set('hwclock', __salt__['timezone.get_hwclock']())
+
+        return pref
+
+    def _get_user_groups(self, user):
+        '''
+        Get user groups.
+
+        :param user:
+        :return:
+        '''
+        return [g.gr_name for g in grp.getgrall()
+                if user in g.gr_mem] + [grp.getgrgid(pwd.getpwnam(user).pw_gid).gr_name]
+
+    def _set_users(self, node):
+        '''
+        Create existing local users.
+
+        <users group="root">
+          <user password="$1$wYJUgpM5$RXMMeASDc035eX.NbYWFl0" home="/root" name="root"/>
+        </users>
+
+        :param node:
+        :return:
+        '''
+        # Get real local users with the local passwords
+        shadow = {}
+        for sh_line in open('/etc/shadow').read().split(os.linesep):
+            if sh_line.strip():
+                login, pwd = sh_line.split(":")[:2]
+                if pwd and pwd[0] not in '!*':
+                    shadow[login] = {'p': pwd}
+
+        for ps_line in open('/etc/passwd').read().split(os.linesep):
+            if ps_line.strip():
+                ps_line = ps_line.strip().split(':')
+                if ps_line[0] in shadow:
+                    shadow[ps_line[0]]['h'] = ps_line[5]
+                    shadow[ps_line[0]]['s'] = ps_line[6]
+                    shadow[ps_line[0]]['g'] = self._get_user_groups(ps_line[0])
+
+        users_groups = []
+        users_node = etree.SubElement(node, 'users')
+        for u_name, u_data in shadow.items():
+            user_node = etree.SubElement(users_node, 'user')
+            user_node.set('password', u_data['p'])
+            user_node.set('home', u_data['h'])
+            user_node.set('name', u_name)
+            users_groups.extend(u_data['g'])
+        users_node.set('group', ','.join(users_groups))
+
+        return users_node
+
+    def _set_repositories(self, node):
+        '''
+        Create repositories.
+
+        :param node:
+        :return:
+        '''
+        priority = 99
+
+        for repo_id, repo_data in self._data.software.get('repositories', {}).items():
+            if type(repo_data) == list:
+                repo_data = repo_data[0]
+            if repo_data.get('enabled') or not repo_data.get('disabled'):  # RPM and Debian, respectively
+                uri = repo_data.get('baseurl', repo_data.get('uri'))
+                if not uri:
+                    continue
+                repo = etree.SubElement(node, 'repository')
+                if self.__grains__.get('os_family') in ('Kali', 'Debian'):
+                    repo.set('alias', repo_id)
+                    repo.set('distribution', repo_data['dist'])
+                else:
+                    repo.set('alias', repo_data['alias'])
+                    if self.__grains__.get('os_family', '') == 'Suse':
+                        repo.set('type', 'yast2')  # TODO: Check for options!
+                    repo.set('priority', str(priority))
+                source = etree.SubElement(repo, 'source')
+                source.set('path', uri)  # RPM and Debian, respectively
+                priority -= 1
+
+    def _set_packages(self, node):
+        '''
+        Set packages and collections.
+
+        :param node:
+        :return:
+        '''
+        pkgs = etree.SubElement(node, 'packages')
+        for pkg_name, pkg_version in sorted(self._data.software.get('packages', {}).items()):
+            pkg = etree.SubElement(pkgs, 'package')
+            pkg.set('name', pkg_name)
+
+        # Add collections (SUSE)
+        if self.__grains__.get('os_family', '') == 'Suse':
+            for ptn_id, ptn_data in self._data.software.get('patterns', {}).items():
+                if ptn_data.get('installed'):
+                    ptn = etree.SubElement(pkgs, 'namedCollection')
+                    ptn.set('name', ptn_id)
+
+        return pkgs
+
+    def _set_description(self, node):
+        '''
+        Create a system description.
+
+        :return:
+        '''
+        hostname = socket.getfqdn() or platform.node()
+
+        descr = etree.SubElement(node, 'description')
+        author = etree.SubElement(descr, 'author')
+        author.text = "salt.modules.node on {0}".format(hostname)
+        contact = etree.SubElement(descr, 'contact')
+        contact.text = 'root@{0}'.format(hostname)
+        specs = etree.SubElement(descr, 'specification')
+        specs.text = 'Rebuild of {0}, based on Salt inspection.'.format(hostname)
+
+        return descr
+
+    def _create_doc(self):
+        '''
+        Create document.
+
+        :return:
+        '''
+        root = etree.Element('image')
+        root.set('schemaversion', '6.3')
+        root.set('name', self.name)
+
+        return root

salt/modules/inspectlib/query.py

@@ -22,8 +22,8 @@ import logging
 
 # Import Salt Libs
 import salt.utils.network
-from salt.modules.inspectlib.dbhandle import DBHandle
 from salt.modules.inspectlib.exceptions import (InspectorQueryException, SIException)
+from salt.modules.inspectlib import EnvLoader
 
 log = logging.getLogger(__name__)
 
@@ -53,8 +53,8 @@ class SysInfo(object):
             log.error(msg)
             raise SIException(msg)
 
-        devpath, blocks, used, available, used_p, mountpoint = [elm for elm in out['stdout'].split(os.linesep)[-1].split(" ") if elm]
-
+        devpath, blocks, used, available, used_p, mountpoint = [elm for elm in
+                                                                out['stdout'].split(os.linesep)[-1].split(" ") if elm]
         return {
             'device': devpath, 'blocks': blocks, 'used': used,
             'available': available, 'used (%)': used_p, 'mounted': mountpoint,
@@ -135,7 +135,7 @@ class SysInfo(object):
         }
 
 
-class Query(object):
+class Query(EnvLoader):
     '''
     Query the system.
     This class is actually puts all Salt features together,
@@ -153,7 +153,7 @@ class Query(object):
 
     SCOPES = ["changes", "configuration", "identity", "system", "software", "services", "payload", "all"]
 
-    def __init__(self, scope):
+    def __init__(self, scope, cachedir=None):
         '''
         Constructor.
 
@@ -163,8 +163,8 @@ class Query(object):
         if scope not in self.SCOPES:
             raise InspectorQueryException(
                 "Unknown scope: {0}. Must be one of: {1}".format(repr(scope), ", ".join(self.SCOPES)))
+        EnvLoader.__init__(self, cachedir=cachedir)
         self.scope = '_' + scope
-        self.db = DBHandle(globals()['__salt__']['config.get']('inspector.db', ''))
         self.local_identity = dict()
 
     def __call__(self, *args, **kwargs):

salt/modules/node.py

@@ -19,8 +19,11 @@ Module for full system inspection.
 '''
 from __future__ import absolute_import
 import logging
+import os
+import getpass
 from salt.modules.inspectlib.exceptions import (InspectorQueryException,
-                                                InspectorSnapshotException)
+                                                InspectorSnapshotException,
+                                                InspectorKiwiProcessorException)
 
 # Import Salt libs
 import salt.utils
@@ -89,7 +92,9 @@ def inspect(mode='all', priority=19, **kwargs):
     '''
     collector = _("collector")
     try:
-        return collector.Inspector().request_snapshot(mode, priority=priority, **kwargs)
+        return collector.Inspector(cachedir=__opts__['cachedir'],
+                                   piddir=os.path.dirname(__opts__['pidfile']))\
+            .request_snapshot(mode, priority=priority, **kwargs)
     except InspectorSnapshotException as ex:
         raise CommandExecutionError(ex)
     except Exception as ex:
@@ -154,9 +159,68 @@ def query(scope, **kwargs):
     '''
     query = _("query")
     try:
-        return query.Query(scope)(**kwargs)
+        return query.Query(scope, cachedir=__opts__['cachedir'])(**kwargs)
     except InspectorQueryException as ex:
         raise CommandExecutionError(ex)
     except Exception as ex:
         log.error(_get_error_message(ex))
         raise Exception(ex)
+
+
+def build(format='qcow2', path='/tmp/'):
+    '''
+    Build an image from a current system description.
+    The image is a system image can be output in bootable ISO or QCOW2 formats.
+
+    Node uses the image building library Kiwi to perform the actual build.
+
+    Parameters:
+
+    * **format**: Specifies output format: "qcow2" or "iso. Default: `qcow2`.
+    * **path**: Specifies output path where to store built image. Default: `/tmp`.
+
+    CLI Example:
+
+    .. code-block:: bash:
+
+        salt myminion node.build
+        salt myminion node.build format=iso path=/opt/builds/
+    '''
+    try:
+        _("collector").Inspector(cachedir=__opts__['cachedir'],
+                                 piddir=os.path.dirname(__opts__['pidfile']),
+                                 pidfilename='').build(format=format, path=path)
+    except InspectorKiwiProcessorException as ex:
+        raise CommandExecutionError(ex)
+    except Exception as ex:
+        log.error(_get_error_message(ex))
+        raise Exception(ex)
+
+
+def export(local=False, path="/tmp", format='qcow2'):
+    '''
+    Export an image description for Kiwi.
+
+    Parameters:
+
+    * **local**: Specifies True or False if the export has to be in the local file. Default: False.
+    * **path**: If `local=True`, then specifies the path where file with the Kiwi description is written.
+                Default: `/tmp`.
+
+    CLI Example:
+
+    .. code-block:: bash:
+
+        salt myminion node.export
+        salt myminion node.export format=iso path=/opt/builds/
+    '''
+    if getpass.getuser() != 'root':
+        raise CommandExecutionError('In order to export system, the minion should run as "root".')
+    try:
+        description = _("query").Query('all', cachedir=__opts__['cachedir'])()
+        return _("collector").Inspector().export(description, local=local, path=path, format=format)
+    except InspectorKiwiProcessorException as ex:
+        raise CommandExecutionError(ex)
+    except Exception as ex:
+        log.error(_get_error_message(ex))
+        raise Exception(ex)

tests/unit/modules/inspect_collector_test.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+'''
+    :codeauthor: :email:`Bo Maryniuk <bo@suse.de>`
+'''
+# Import Python Libs
+from __future__ import absolute_import
+import os
+
+# Import Salt Testing Libs
+from salttesting import TestCase, skipIf
+from salttesting.mock import (
+    MagicMock,
+    patch,
+    NO_MOCK,
+    NO_MOCK_REASON
+)
+
+from salt.modules.inspectlib.collector import Inspector
+from salttesting.helpers import ensure_in_syspath
+
+ensure_in_syspath('../../')
+
+
+@skipIf(NO_MOCK, NO_MOCK_REASON)
+class InspectorCollectorTestCase(TestCase):
+    '''
+    Test inspectlib:collector:Inspector
+    '''
+    def test_env_loader(self):
+        '''
+        Get packages on the different distros.
+
+        :return:
+        '''
+        inspector = Inspector(cachedir='/foo/cache', piddir='/foo/pid', pidfilename='bar.pid')
+        self.assertEqual(inspector.dbfile, '/foo/cache/_minion_collector.db')
+        self.assertEqual(inspector.pidfile, '/foo/pid/bar.pid')
+
+    def test_file_tree(self):
+        '''
+        Test file tree.
+
+        :return:
+        '''
+
+        inspector = Inspector(cachedir='/test', piddir='/test', pidfilename='bar.pid')
+        tree_root = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'inspectlib', 'tree_test')
+        expected_tree = (['/a/a/dummy.a', '/a/b/dummy.b', '/b/b.1', '/b/b.2', '/b/b.3'],
+                         ['/a', '/a/a', '/a/b', '/a/c', '/b', '/c'],
+                         ['/a/a/dummy.ln.a', '/a/b/dummy.ln.b', '/a/c/b.1', '/b/b.4',
+                          '/b/b.5', '/c/b.1', '/c/b.2', '/c/b.3'])
+        tree_result = []
+        for chunk in inspector._get_all_files(tree_root):
+            buff = []
+            for pth in chunk:
+                buff.append(pth.replace(tree_root, ''))
+            tree_result.append(buff)
+        tree_result = tuple(tree_result)
+        self.assertEqual(expected_tree, tree_result)
+
+    def test_get_unmanaged_files(self):
+        '''
+        Test get_unmanaged_files.
+
+        :return:
+        '''
+        inspector = Inspector(cachedir='/test', piddir='/test', pidfilename='bar.pid')
+        managed = (
+            ['a', 'b', 'c'],
+            ['d', 'e', 'f'],
+            ['g', 'h', 'i'],
+        )
+        system_all = (
+            ['a', 'b', 'c'],
+            ['d', 'E', 'f'],
+            ['G', 'H', 'i'],
+        )
+        self.assertEqual(inspector._get_unmanaged_files(managed=managed, system_all=system_all),
+                         ([], ['E'], ['G', 'H']))
+
+    def test_pkg_get(self):
+        '''
+        Test if grains switching the pkg get method.
+
+        :return:
+        '''
+        debian_list = """
+g++
+g++-4.9
+g++-5
+gawk
+gcc
+gcc-4.9
+gcc-4.9-base:amd64
+gcc-4.9-base:i386
+gcc-5
+gcc-5-base:amd64
+gcc-5-base:i386
+gcc-6-base:amd64
+gcc-6-base:i386
+"""
+        inspector = Inspector(cachedir='/test', piddir='/test', pidfilename='bar.pid')
+        inspector.grains_core = MagicMock()
+        inspector.grains_core.os_data = MagicMock()
+        inspector.grains_core.os_data.get = MagicMock(return_value='Debian')
+        with patch.object(inspector, '_Inspector__get_cfg_pkgs_dpkg', MagicMock(return_value='dpkg')):
+            with patch.object(inspector, '_Inspector__get_cfg_pkgs_rpm', MagicMock(return_value='rpm')):
+                inspector.grains_core = MagicMock()
+                inspector.grains_core.os_data = MagicMock()
+                inspector.grains_core.os_data().get = MagicMock(return_value='Debian')
+                self.assertEqual(inspector._get_cfg_pkgs(), 'dpkg')
+                inspector.grains_core.os_data().get = MagicMock(return_value='Suse')
+                self.assertEqual(inspector._get_cfg_pkgs(), 'rpm')
+                inspector.grains_core.os_data().get = MagicMock(return_value='redhat')
+                self.assertEqual(inspector._get_cfg_pkgs(), 'rpm')

tests/unit/modules/inspectlib/tree_test/a/a/dummy.ln.a

@@ -0,0 +1 @@
+dummy.a

tests/unit/modules/inspectlib/tree_test/a/b/dummy.ln.b

@@ -0,0 +1 @@
+dummy.b

tests/unit/modules/inspectlib/tree_test/a/c/b.1

@@ -0,0 +1 @@
+../../b/b.1

tests/unit/modules/inspectlib/tree_test/b/b.1

@@ -0,0 +1 @@
+B.1

tests/unit/modules/inspectlib/tree_test/b/b.4

@@ -0,0 +1 @@
+../c/b.1

tests/unit/modules/inspectlib/tree_test/b/b.5

@@ -0,0 +1 @@
+b.4

tests/unit/modules/inspectlib/tree_test/c/b.1

@@ -0,0 +1 @@
+../b/b.1

tests/unit/modules/inspectlib/tree_test/c/b.2

@@ -0,0 +1 @@
+../b/b.2

tests/unit/modules/inspectlib/tree_test/c/b.3

@@ -0,0 +1 @@
+../b/b.3