From 06f2bba18aba0dd4aef7957c1cb3c9048bf0a371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20B=C3=A9rtoli?= Date: Mon, 12 Feb 2018 08:16:03 -0300 Subject: [PATCH] Initial commit --- .gitignore | 4 +- .kitchen.yml | 77 +++++++++++++ CHANGELOG.rst | 6 ++ README.md | 2 - README.rst | 101 ++++++++++++++++++ packages/defaults.yaml | 22 ++++ packages/gems.sls | 35 ++++++ packages/init.sls | 8 ++ packages/map.jinja | 15 +++ packages/osfamilymap.yaml | 25 +++++ packages/pips.sls | 35 ++++++ packages/pkgs.sls | 42 ++++++++ packages/remote_pkgs.sls | 16 +++ pillar.example | 32 ++++++ .../integration/default/pillar.example.redhat | 31 ++++++ test/integration/default/pips_spec.rb | 18 ++++ test/integration/default/pkgs_spec.rb | 20 ++++ test/integration/default/remote_pkgs_spec.rb | 7 ++ 18 files changed, 493 insertions(+), 3 deletions(-) create mode 100644 .kitchen.yml create mode 100644 CHANGELOG.rst delete mode 100644 README.md create mode 100644 README.rst create mode 100644 packages/defaults.yaml create mode 100644 packages/gems.sls create mode 100644 packages/init.sls create mode 100644 packages/map.jinja create mode 100644 packages/osfamilymap.yaml create mode 100644 packages/pips.sls create mode 100644 packages/pkgs.sls create mode 100644 packages/remote_pkgs.sls create mode 100644 pillar.example create mode 100644 test/integration/default/pillar.example.redhat create mode 100644 test/integration/default/pips_spec.rb create mode 100644 test/integration/default/pkgs_spec.rb create mode 100644 test/integration/default/remote_pkgs_spec.rb diff --git a/.gitignore b/.gitignore index 7bbc71c..45f46f6 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,7 @@ wheels/ *.egg # PyInstaller -# Usually these files are written by a python script from a template +# Usually these files are written by a python script from a packager # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec @@ -45,6 +45,8 @@ nosetests.xml coverage.xml *.cover .hypothesis/ +.kitchen +.kitchen.local.yml # Translations *.mo diff --git a/.kitchen.yml b/.kitchen.yml new file mode 100644 index 0000000..1b0837b --- /dev/null +++ b/.kitchen.yml @@ -0,0 +1,77 @@ +--- +driver: + name: docker + +driver_config: + use_sudo: false + privileged: true + provision_command: mkdir -p /run/sshd + +platforms: + - name: debian-9 + driver_config: + image: nm/debian + - name: centos-7 + driver_config: + image: saltstack/centos-7-minimal + - name: ubuntu-17.10 + +provisioner: + name: salt_solo + log_level: info + require_chef: false + salt_version: latest + formula: packages + salt_copy_filter: + - .kitchen + - .git + pillars-from-files: + packages.sls: pillar.example + pillars: + top.sls: + base: + '*': + - packages + +verifier: + name: inspec + sudo: false + # cli, documentation, html, progress, json, json-min, json-rspec, junit + format: cli + inspec_tests: + - path: test/integration/default + +suites: + - name: deb + provisioner: + state_top: + base: + '*': + - packages + excludes: + - centos-7 + - name: rpm + provisioner: + dependencies: + - name: epel + repo: git + source: https://github.com/saltstack-formulas/epel-formula.git + state_top: + base: + '*': + - epel + - packages + pillars-from-files: + packages.sls: test/integration/default/pillar.example.redhat + pillars: + top.sls: + base: + '*': + - epel + - packages + epel.sls: + disabled: false + excludes: + - debian-9 + - ubuntu-17.10 + diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..9208e68 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,6 @@ +packages formula +================ + +0.0.1 (2018-02-12) + +- Initial version diff --git a/README.md b/README.md deleted file mode 100644 index 2efda19..0000000 --- a/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# packager-formula -Simple management of wanted/unwanted packages/gems/eggs diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..74e6fdd --- /dev/null +++ b/README.rst @@ -0,0 +1,101 @@ +================ +packages-formula +================ + +A simple 'packages manager' formula, to install/remove packages without further +ado. + +What this formula can do +======================== + +Many times, in the process of setting up a host/vm, you need to install/remove +packages with no extra configuration or setup. This formula tries to help with +that. It can get a list of packages from a pillar, and it will try to install +them. + +It provides a few states to install/remove system packages (currently +Debian/Redhat families), Python packages (using pip states) and Ruby gems +(using gem states). + +It can also provide basic dependency management on certain other states/packages. + +What this formula can't do +========================== + +This formula is not intended to configure packages, nor setup services or daemons. +When you need to do that for a package, you should probably be using another +formula. + +.. note:: + + See the full `Salt Formulas installation and usage instructions + `_. + +Available states +================ + +.. contents:: + :local: + +``packages`` +--=--------- + +Runs all the other states in the formula. + +``packages.pkgs`` +----------------- + +You can specify: + +* ``wanted`` system packages, which will be installed. +* ``unwanted`` system packages, which will be uninstalled. +* ``required system packages`` on which any of the ``wanted`` packages depend + for their correct installation. +* ``required states`` on which any of the ``wanted`` packages depend for their + correct installation. + +``packages.pips`` +----------------- + +You can specify: + +* ``wanted`` python packages, which will be installed using pip. Requires you + specify the correct ``python-pip`` package for your distro, as a dependency + (see the pillar.example) +* ``unwanted`` python packages, which will be uninstalled using pip. +* ``required system packages`` on which any of the ``wanted`` python packages + depend for their correct installation. Usually, a ``python-pip`` package and/or + some other compiler packages are required. +* ``required states`` on which any of the ``wanted`` packages depend for their + correct installation (ie, ``epel`` for RedHat families). + +``packages.gems`` +----------------- + +You can specify: + +* ``wanted`` ruby packages, which will be installed using gem. Requires you + specify the correct ``ruby`` package for your distro, as a dependency + (see the pillar.example) +* ``unwanted`` ruby packages, which will be uninstalled using gem. +* ``required system packages`` on which any of the ``wanted`` ruby packages + depend for their correct installation. Usually, a ``ruby`` package and/or + some other compiler packages are required. +* ``required states`` on which any of the ``wanted`` packages depend for their + correct installation (ie, ``epel`` for RedHat families). + +``packages.remote_pkgs`` +------------------------ + +You can specify a dictionary of remote system packages (deb/rpm) that you want +to install, in the format: + +``name: url`` + +Testing +======= + +Testing is done with `Test Kitchen `_ +for machine setup and `testinfra `_ +for integration tests. + diff --git a/packages/defaults.yaml b/packages/defaults.yaml new file mode 100644 index 0000000..15a12b5 --- /dev/null +++ b/packages/defaults.yaml @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +packages: + remote_pkgs: {} + pkgs: + wanted: [] + unwanted: [] + required: + states: [] + pkgs: [] + pips: + wanted: [] + unwanted: [] + required: + states: [] + pkgs: [] + gems: + wanted: [] + unwanted: [] + required: + states: [] + pkgs: [] diff --git a/packages/gems.sls b/packages/gems.sls new file mode 100644 index 0000000..03f731b --- /dev/null +++ b/packages/gems.sls @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls +{% from "packages/map.jinja" import packages with context %} + +{% set req_states = packages.gems.required.states %} +{% set req_pkgs = packages.gems.required.pkgs %} +{% set wanted_gems = packages.gems.wanted %} +{% set unwanted_gems = packages.gems.unwanted %} + +### REQ PKGS (without these, some of the WANTED GEMS will fail to install) +{% if req_pkgs != {} %} +gem_req_pkgs: + pkg.installed: + - pkgs: {{ req_pkgs }} +{% endif %} + +### GEMS to install +# (requires the ruby/rubygem deb/rpm installed, either by the system or listed in +# the required packages +{% for gm in wanted_gems %} +{{ gm }}: + gem.installed: + - require: + - pkg: gem_req_pkgs + {% if req_states %} + {% for dep in req_states %} + - sls: {{ dep }} + {% endfor %} + {% endif %} +{% endfor %} + +{% for ugm in unwanted_gems %} +{{ ugm }}: + gem.removed +{% endfor %} diff --git a/packages/init.sls b/packages/init.sls new file mode 100644 index 0000000..e96d5dd --- /dev/null +++ b/packages/init.sls @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +include: + - packages.pkgs + - packages.remote_pkgs + - packages.pips + - packages.gems diff --git a/packages/map.jinja b/packages/map.jinja new file mode 100644 index 0000000..0aa7a73 --- /dev/null +++ b/packages/map.jinja @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# vim: ft=jinja + +{% import_yaml 'packages/defaults.yaml' as defaults %} +{% import_yaml 'packages/osfamilymap.yaml' as osfamilymap %} + +{% set packages = salt['grains.filter_by']( + defaults, + merge=salt['grains.filter_by']( + osfamilymap, + grain='os_family', + merge=salt['pillar.get']('packages', {}), + ), + base='packages') +%} diff --git a/packages/osfamilymap.yaml b/packages/osfamilymap.yaml new file mode 100644 index 0000000..a6fcb88 --- /dev/null +++ b/packages/osfamilymap.yaml @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +Debian: + pips: + required: + pkgs: + - python-pip + gems: + required: + pkgs: + - ruby +RedHat: + pips: + required: + states: + - epel + pkgs: + - gcc + - python-devel + - python2-pip + gems: + required: + pkgs: + - rubygems + diff --git a/packages/pips.sls b/packages/pips.sls new file mode 100644 index 0000000..29bb416 --- /dev/null +++ b/packages/pips.sls @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls +{% from "packages/map.jinja" import packages with context %} + +{% set req_states = packages.pips.required.states %} +{% set req_pkgs = packages.pips.required.pkgs %} +{% set wanted_pips = packages.pips.wanted %} +{% set unwanted_pips = packages.pips.unwanted %} + +### REQ PKGS (without these, some of the WANTED PIPS will fail to install) +{% if req_pkgs != {} %} +pip_req_pkgs: + pkg.installed: + - pkgs: {{ req_pkgs }} +{% endif %} + +### PYTHON PKGS to install using PIP +# (requires the python-pip deb/rpm installed, either by the system or listed in +# the required packages +{% for pn in wanted_pips %} +{{ pn }}: + pip.installed: + - require: + - pkg: pip_req_pkgs + {% if req_states %} + {% for dep in req_states %} + - sls: {{ dep }} + {% endfor %} + {% endif %} +{% endfor %} + +{% for upn in unwanted_pips %} +{{ upn }}: + pip.removed +{% endfor %} diff --git a/packages/pkgs.sls b/packages/pkgs.sls new file mode 100644 index 0000000..b549a72 --- /dev/null +++ b/packages/pkgs.sls @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{% from "packages/map.jinja" import packages with context %} + +{% set req_states = packages.pkgs.required.states %} +{% set req_packages = packages.pkgs.required.pkgs %} +{% set wanted_packages = packages.pkgs.wanted %} +{% set unwanted_packages = packages.pkgs.unwanted %} + +### PRE-REQ PKGS (without these, some of the WANTED PKGS will fail to install) +{% if req_packages != {} %} +prereq_packages: + pkg.installed: + - pkgs: {{ req_packages }} + {% if req_states %} + - require: + {% for dep in req_states %} + - sls: {{ dep }} + {% endfor %} + {% endif %} +{% endif %} + +{% if wanted_packages != {} %} +wanted_packages: + pkg.installed: + - pkgs: {{ wanted_packages }} + - require: + - pkg: prereq_packages + {% if req_states %} + {% for dep in req_states %} + - sls: {{ dep }} + {% endfor %} + {% endif %} +{% endif %} + +{% if unwanted_packages != {} %} +unwanted_packages: + pkg.purged: + - pkgs: {{ unwanted_packages }} +{% endif %} + diff --git a/packages/remote_pkgs.sls b/packages/remote_pkgs.sls new file mode 100644 index 0000000..da1aa4c --- /dev/null +++ b/packages/remote_pkgs.sls @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{% from "packages/map.jinja" import packages with context %} + +{% set wanted_rem_pkgs = packages.remote_pkgs %} + +### REMOTE PKGS to install directly from a remote URL. Must be a deb/rpm file +{% if wanted_rem_pkgs != {} %} +remote_packages: + pkg.installed: + - sources: + {% for package, url in wanted_rem_pkgs.items() %} + - {{ package }}: {{ url }} + {% endfor %} +{% endif %} diff --git a/pillar.example b/pillar.example new file mode 100644 index 0000000..4636a96 --- /dev/null +++ b/pillar.example @@ -0,0 +1,32 @@ +packages: + pkgs: + wanted: + - git + - less + - bc + - curl + - fail2ban + unwanted: + - avahi-daemon + required: + pkgs: + - git + pips: + wanted: + - dxpy + - makerlabs + unwanted: + - campbel + - reverse_geocode + - indy-crypto + gems: + wanted: + - progressbar + - aws-sdk + unwanted: + - diff-lcs + - kitchen-vagrant + - kwalify + remote_pkgs: + zoom: 'https://zoom.us/client/latest/zoom_amd64.deb' + diff --git a/test/integration/default/pillar.example.redhat b/test/integration/default/pillar.example.redhat new file mode 100644 index 0000000..611121c --- /dev/null +++ b/test/integration/default/pillar.example.redhat @@ -0,0 +1,31 @@ +packages: + pkgs: + wanted: + - git + - less + - bc + - curl + - fail2ban + unwanted: + - avahi-daemon + required: + pkgs: + - git + pips: + wanted: + - dxpy + - makerlabs + unwanted: + - campbel + - reverse_geocode + - indy-crypto + gems: + wanted: + - progressbar + - aws-sdk + unwanted: + - diff-lcs + - kitchen-vagrant + - kwalify + remote_pkgs: + zoom: 'https://zoom.us/client/latest/zoom_x86_64.rpm' diff --git a/test/integration/default/pips_spec.rb b/test/integration/default/pips_spec.rb new file mode 100644 index 0000000..390f24d --- /dev/null +++ b/test/integration/default/pips_spec.rb @@ -0,0 +1,18 @@ +%w{ + dxpy + makerlabs +}.each do |p| + describe pip(p) do + it { should be_installed } + end +end + +%w{ + campbel + reverse_geocode + indy-crypto +}.each do |p| + describe pip(p) do + it { should_not be_installed } + end +end diff --git a/test/integration/default/pkgs_spec.rb b/test/integration/default/pkgs_spec.rb new file mode 100644 index 0000000..27de78b --- /dev/null +++ b/test/integration/default/pkgs_spec.rb @@ -0,0 +1,20 @@ +%w{ + git + git + less + bc + curl + fail2ban +}.each do |p| + describe package(p) do + it { should be_installed } + end +end + +%w{ + avahi-daemon +}.each do |p| + describe package(p) do + it { should_not be_installed } + end +end diff --git a/test/integration/default/remote_pkgs_spec.rb b/test/integration/default/remote_pkgs_spec.rb new file mode 100644 index 0000000..2414948 --- /dev/null +++ b/test/integration/default/remote_pkgs_spec.rb @@ -0,0 +1,7 @@ +%w{ + zoom +}.each do |p| + describe package(p) do + it { should be_installed } + end +end