From ce9826dcdce2f657e8a1d7385e50fc660c076d17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20B=C3=A9rtoli?= Date: Sun, 10 Mar 2019 09:18:39 -0300 Subject: [PATCH 1/5] Add npm management support --- CHANGELOG.rst | 4 +++ README.rst | 22 ++++++++++++++++ kitchen.yml | 30 +++++++++++++++++++--- packages/defaults.yaml | 6 +++++ packages/init.sls | 1 + packages/npms.sls | 34 +++++++++++++++++++++++++ pillar.example | 14 +++++++++++ test/integration/default/npms_spec.rb | 36 +++++++++++++++++++++++++++ 8 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 packages/npms.sls create mode 100644 test/integration/default/npms_spec.rb diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 14d8a88..4a48855 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -37,3 +37,7 @@ packages formula 0.0.9 (2018-10-03) - Add support for plain files using arcihve + +0.1.0 (2019-03-10) + +- Add npm support diff --git a/README.rst b/README.rst index 7e6361f..1d8e3df 100644 --- a/README.rst +++ b/README.rst @@ -127,6 +127,28 @@ You can specify: * ``required states`` on which any of the ``wanted`` packages depend for their correct installation (ie, ``epel`` for RedHat families). +``packages.npms`` +----------------- + +This formula **DOES NOT** install ``nodejs/npm``, as it's outside of its scope: +nodejs/npm that comes with the distros is usually outdated, so it's required to add +a repo, run scripts, etc, and this formula manages packages :) + +You can use the `nodejs-formula `_ +and add a dependency for it in the pillar `npms:required:sls` (see the pillar.example) + +You can specify: + +* ``wanted`` npm packages, which will be installed using npm. Requires you + specify the correct ``npm`` package for your distro, as a dependency + (see the pillar.example) +* ``unwanted`` npm packages, which will be uninstalled using npm. +* ``required system packages`` on which any of the ``wanted`` npm packages + depend for their correct installation. Usually, a ``npm`` package and/or + some other compiler packages are required. +* ``required states`` on which any of the ``wanted`` packages depend for their + correct installation (ie, ``epel`` for RedHat families). + ``packages.archives`` ------------------- diff --git a/kitchen.yml b/kitchen.yml index 95fd485..6032baf 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -20,19 +20,19 @@ platforms: image: debian:9 run_command: /lib/systemd/systemd provision_command: - - apt-get update && apt-get install -y udev locales + - apt-get update && apt-get install -y udev locales git - name: ubuntu-18.04 driver_config: image: ubuntu:18.04 run_command: /lib/systemd/systemd provision_command: - - apt-get update && apt-get install -y udev locales + - apt-get update && apt-get install -y udev locales git - name: ubuntu-16.04 driver_config: image: ubuntu:16.04 run_command: /lib/systemd/systemd provision_command: - - apt-get update && apt-get install -y udev locales + - apt-get update && apt-get install -y udev locales git - locale-gen en_US.UTF-8 - update-locale LANG=en_US.UTF-8 - name: centos-7 @@ -106,6 +106,30 @@ suites: - centos-7 - fedora - opensuse-leap-salt-minion + provisioner: + dependencies: + - name: node + repo: git + source: https://github.com/saltstack-formulas/node-formula.git + state_top: + base: + '*': + - node + - packages + pillars_from_files: + packages.sls: pillar.example + pillars: + top.sls: + base: + '*': + - node + - packages + node.sls: + node: + version: 11.11.0-1nodesource1 + install_from_ppa: True + ppa: + repository_url: https://deb.nodesource.com/node_11.x - name: ubu18 excludes: diff --git a/packages/defaults.yaml b/packages/defaults.yaml index 8bf4520..6ef3638 100644 --- a/packages/defaults.yaml +++ b/packages/defaults.yaml @@ -23,6 +23,12 @@ packages: required: states: [] pkgs: [] + npms: + wanted: [] + unwanted: [] + required: + states: [] + pkgs: [] snaps: packages: ['snapd', 'fuse',] collides: [] diff --git a/packages/init.sls b/packages/init.sls index 2438cb3..b273e2c 100644 --- a/packages/init.sls +++ b/packages/init.sls @@ -6,5 +6,6 @@ include: - packages.remote_pkgs - packages.pips - packages.gems + - packages.npms - packages.archives - packages.snaps diff --git a/packages/npms.sls b/packages/npms.sls new file mode 100644 index 0000000..4828cc3 --- /dev/null +++ b/packages/npms.sls @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls +{% from "packages/map.jinja" import packages with context %} + +{% set req_states = packages.npms.required.states %} +{% set req_pkgs = packages.npms.required.pkgs %} +{% set wanted_npms = packages.npms.wanted %} +{% set unwanted_npms = packages.npms.unwanted %} + +### REQ PKGS (without these, some of the WANTED PIPS will fail to install) +npm_req_pkgs: + pkg.installed: + - pkgs: {{ req_pkgs | json }} + +### NPM PKGS to install using npm +# (requires the npm deb/rpm installed, either by the system or listed in +# the required packages +{% for nn in wanted_npms %} +{{ nn }}: + npm.installed: + - reload_modules: true + - require: + - pkg: npm_req_pkgs + {% if req_states %} + {% for dep in req_states %} + - sls: {{ dep }} + {% endfor %} + {% endif %} +{% endfor %} + +{% for upn in unwanted_npms %} +{{ upn }}: + npm.removed +{% endfor %} diff --git a/pillar.example b/pillar.example index b2f950f..c61f94b 100644 --- a/pillar.example +++ b/pillar.example @@ -43,6 +43,7 @@ packages: timeout: 120 default-timeout: 120 # proxy: http://proxy.example.com:3128 + gems: wanted: - progressbar @@ -51,6 +52,7 @@ packages: - diff-lcs - kitchen-vagrant - kwalify + snaps: wanted: - hello-world @@ -58,6 +60,18 @@ packages: - test-snapd-hello-classic unwanted: - goodbye-world + + npms: + required: + # pkgs: + # - some_package_you_need + sls: ['nodejs.ppa'] + wanted: + - hello-world-npm + - sax + unwanted: + - gist + archives: wanted: terminator: diff --git a/test/integration/default/npms_spec.rb b/test/integration/default/npms_spec.rb new file mode 100644 index 0000000..5077b8d --- /dev/null +++ b/test/integration/default/npms_spec.rb @@ -0,0 +1,36 @@ +### WANTED/REQUIRED +control 'Wanted/Required npm packages' do + title 'should be installed' + desc '(only testing in the Debian platform, as the node-formual dependency is too specific)' + + only_if do + os.name == 'debian' + end + + %w{ + sax + hello-world-npm + }.each do |p| + describe npm(p) do + it { should be_installed } + end + end +end + +### UNWANTED +control 'Unwanted npm packages' do + title 'should be uninstalled' + desc '(only testing in the Debian platform, as the node-formual dependency is too specific)' + + only_if do + os.name == 'debian' + end + + %w{ + gist + }.each do |p| + describe npm(p) do + it { should_not be_installed } + end + end +end From dbae8025212825446af8dec2a8b407a0954b4f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20B=C3=A9rtoli?= Date: Sun, 10 Mar 2019 22:21:43 -0300 Subject: [PATCH 2/5] Allow to specify path/user for npms --- CHANGELOG.rst | 58 +++++++++++++-------------- packages/npms.sls | 37 ++++++++++++++--- pillar.example | 19 +++++++-- test/integration/default/npms_spec.rb | 18 ++++++--- 4 files changed, 89 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4a48855..ddaa4f0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,43 +1,43 @@ packages formula ================ -0.0.1 (2018-02-12) +0.1.0 (2019-03-10) -- Initial version +- Add npm support -0.0.2 (2018-02-23) +0.0.9 (2018-10-03) -- Add Fedora support - -0.0.3 (2018-03-02) - -- Allow to hold/unhold system packages (#8) -- Add more tests - -0.0.4 (2018-03-06) - -- Allow to specify held system packages also as a list (#10) - -0.0.5 (2018-03-14) - -- Add snap support - -0.0.6 (2018-04-09) - -- Add snap with classic confinement support - -0.0.7 (2018-07-11) - -- Fix pip/gems for FreeBSD +- Add support for plain files using archive 0.0.8 (2018-08-23) - Add archive support -0.0.9 (2018-10-03) +0.0.7 (2018-07-11) -- Add support for plain files using arcihve +- Fix pip/gems for FreeBSD -0.1.0 (2019-03-10) +0.0.6 (2018-04-09) -- Add npm support +- Add snap with classic confinement support + +0.0.5 (2018-03-14) + +- Add snap support + +0.0.4 (2018-03-06) + +- Allow to specify held system packages also as a list (#10) + +0.0.3 (2018-03-02) + +- Allow to hold/unhold system packages (#8) + +0.0.2 (2018-02-23) + +- Add Fedora support +- Add more tests + +0.0.1 (2018-02-12) + +- Initial version diff --git a/packages/npms.sls b/packages/npms.sls index 4828cc3..4bea20a 100644 --- a/packages/npms.sls +++ b/packages/npms.sls @@ -13,20 +13,47 @@ npm_req_pkgs: - pkgs: {{ req_pkgs | json }} ### NPM PKGS to install using npm -# (requires the npm deb/rpm installed, either by the system or listed in +# (requires the npm binary installed, either by the system or listed in # the required packages -{% for nn in wanted_npms %} -{{ nn }}: + +{% if packages.npms.dir is defined %} +npms_dir: + file.directory: + - name: {{ packages.npms.dir }} + - user: {{ 'root' if 'user' not in packages.npms else packages.npms.user }} + - group: {{ 'root' if 'group' not in packages.npms else packages.npms.group }} + - mode: {{ '0755' if 'mode' not in packages.npms else packages.npms.mode }} + - makedirs: True +{% endif %} + +wanted_npms: npm.installed: - - reload_modules: true + - pkgs: {{ wanted_npms | json }} + {% if packages.npms.dir is defined %} + - dir: {{ packages.npms.dir }} + {% endif %} + {% if packages.npms.user is defined %} + - user: {{ packages.npms.user }} + {% endif %} + {% if packages.npms.registry is defined %} + - registry: {{ packages.npms.registry }} + {% endif %} + {% if packages.npms.env is defined %} + - env: {{ packages.npms.env | json }} + {% endif %} + {% if packages.npms.force_reinstall is defined %} + - force_reinstall: {{ packages.npms.force_reinstall }} + {% endif %} - require: + {% if packages.npms.dir is defined %} + - file: npms_dir + {% endif %} - pkg: npm_req_pkgs {% if req_states %} {% for dep in req_states %} - sls: {{ dep }} {% endfor %} {% endif %} -{% endfor %} {% for upn in unwanted_npms %} {{ upn }}: diff --git a/pillar.example b/pillar.example index c61f94b..92d25f6 100644 --- a/pillar.example +++ b/pillar.example @@ -62,13 +62,26 @@ packages: - goodbye-world npms: + # dir: /home/kitchen/npms # The target directory in which to install the package, or None for global installation + # user: kitchen # The user to run NPM with (and to assign to `dir`) + # group: kitchen # The group to assign to `dir` + # mode: 0755 # The permissions to assign to `dir` + # registry: None # The NPM registry from which to install the package + # env: None # A list of environment variables to be set prior to execution + # force_reinstall: False # Install the package even if it is already installed required: - # pkgs: - # - some_package_you_need - sls: ['nodejs.ppa'] + pkgs: + - nodejs wanted: + # Valid formats: + # + # @google-cloud/bigquery@^0.9.6 + # @foobar + # buffer-equal-constant-time@1.0.1 + # coffee-script - hello-world-npm - sax + - coffee-script@1.0.1 unwanted: - gist diff --git a/test/integration/default/npms_spec.rb b/test/integration/default/npms_spec.rb index 5077b8d..debcb46 100644 --- a/test/integration/default/npms_spec.rb +++ b/test/integration/default/npms_spec.rb @@ -1,18 +1,24 @@ +wanted_npms = { + 'hello-world-npm': '1.1.1', + 'sax': '1.2.4', + 'coffee-script': '1.0.1' +} + ### WANTED/REQUIRED control 'Wanted/Required npm packages' do title 'should be installed' - desc '(only testing in the Debian platform, as the node-formual dependency is too specific)' + desc '(only testing in the Debian platform, as the node-formula dependency is too specific)' only_if do os.name == 'debian' end - %w{ - sax - hello-world-npm - }.each do |p| + wanted_npms.each do |p,v| describe npm(p) do it { should be_installed } + # FIXME! Testing for version is failing, seems an issue in inspec + # Same happens with testing path, as it performs a cd command and it fails when using sudo + # its('version') { should eq v } end end end @@ -20,7 +26,7 @@ end ### UNWANTED control 'Unwanted npm packages' do title 'should be uninstalled' - desc '(only testing in the Debian platform, as the node-formual dependency is too specific)' + desc '(only testing in the Debian platform, as the node-formula dependency is too specific)' only_if do os.name == 'debian' From df99e1cdcca1f11158bc03081a1a41673c6f1842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20B=C3=A9rtoli?= Date: Sun, 10 Mar 2019 22:47:54 -0300 Subject: [PATCH 3/5] Fix dependency on states As per https://docs.saltstack.com/en/latest/ref/states/requisites.html#require, the `require` dependency on a sls file needs an include, which was missing --- packages/gems.sls | 7 +++++++ packages/npms.sls | 7 +++++++ packages/pips.sls | 7 +++++++ packages/pkgs.sls | 7 +++++++ packages/snaps.sls | 5 +++++ pillar.example | 4 ++-- 6 files changed, 35 insertions(+), 2 deletions(-) diff --git a/packages/gems.sls b/packages/gems.sls index df98299..2fd2b00 100644 --- a/packages/gems.sls +++ b/packages/gems.sls @@ -7,6 +7,13 @@ {% set wanted_gems = packages.gems.wanted %} {% set unwanted_gems = packages.gems.unwanted %} +{% if req_states %} +include: + {% for dep in req_states %} + - {{ dep }} + {% endfor %} +{% endif %} + ### REQ PKGS (without these, some of the WANTED GEMS will fail to install) gem_req_pkgs: pkg.installed: diff --git a/packages/npms.sls b/packages/npms.sls index 4bea20a..f57a7a0 100644 --- a/packages/npms.sls +++ b/packages/npms.sls @@ -7,6 +7,13 @@ {% set wanted_npms = packages.npms.wanted %} {% set unwanted_npms = packages.npms.unwanted %} +{% if req_states %} +include: + {% for dep in req_states %} + - {{ dep }} + {% endfor %} +{% endif %} + ### REQ PKGS (without these, some of the WANTED PIPS will fail to install) npm_req_pkgs: pkg.installed: diff --git a/packages/pips.sls b/packages/pips.sls index 45cd6ae..064abb3 100644 --- a/packages/pips.sls +++ b/packages/pips.sls @@ -8,6 +8,13 @@ {% set unwanted_pips = packages.pips.unwanted %} {% set pip_config = packages.pips.config %} +{% if req_states %} +include: + {% for dep in req_states %} + - {{ dep }} + {% endfor %} +{% endif %} + ### REQ PKGS (without these, some of the WANTED PIPS will fail to install) pip_req_pkgs: pkg.installed: diff --git a/packages/pkgs.sls b/packages/pkgs.sls index 06b77a3..8948c77 100644 --- a/packages/pkgs.sls +++ b/packages/pkgs.sls @@ -9,6 +9,13 @@ {% set wanted_packages = packages.pkgs.wanted %} {% set unwanted_packages = packages.pkgs.unwanted %} +{% if req_states %} +include: + {% for dep in req_states %} + - {{ dep }} + {% endfor %} +{% endif %} + ### PRE-REQ PKGS (without these, some of the WANTED PKGS will fail to install) pkg_req_pkgs: pkg.installed: diff --git a/packages/snaps.sls b/packages/snaps.sls index 66b8c3d..1c65134 100644 --- a/packages/snaps.sls +++ b/packages/snaps.sls @@ -18,6 +18,11 @@ ### REQ PKGS (without this, SNAPS can fail to install/uninstall) include: - packages.pkgs + {% if req_states %} + {% for dep in req_states %} + - {{ dep }} + {% endfor %} + {% endif %} extend: unwanted_pkgs: diff --git a/pillar.example b/pillar.example index 92d25f6..9212648 100644 --- a/pillar.example +++ b/pillar.example @@ -70,8 +70,8 @@ packages: # env: None # A list of environment variables to be set prior to execution # force_reinstall: False # Install the package even if it is already installed required: - pkgs: - - nodejs + states: + - node.pkg wanted: # Valid formats: # From 9c83969acc00ca933b35eb15e1434212fc9ddcbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20B=C3=A9rtoli?= Date: Sun, 10 Mar 2019 23:36:03 -0300 Subject: [PATCH 4/5] Add a conditional to run npms only when explicitely required --- packages/npms.sls | 9 ++++++++- pillar.example | 8 ++++---- test/integration/default/npms_spec.rb | 19 ++++++++++++++----- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/packages/npms.sls b/packages/npms.sls index f57a7a0..6315965 100644 --- a/packages/npms.sls +++ b/packages/npms.sls @@ -14,7 +14,12 @@ include: {% endfor %} {% endif %} -### REQ PKGS (without these, some of the WANTED PIPS will fail to install) +# As we depend on npm installed, if this state file is invoked every time +# if will fail with 'npm not found'. This condition makes sure it's run +# only when explicitly asking for adding/removing npms +{% if wanted_npms or unwanted_npms %} + +### REQ PKGS (without these, some of the WANTED NPMS will fail to install) npm_req_pkgs: pkg.installed: - pkgs: {{ req_pkgs | json }} @@ -66,3 +71,5 @@ wanted_npms: {{ upn }}: npm.removed {% endfor %} + +{% endif %} diff --git a/pillar.example b/pillar.example index 9212648..17d1b4b 100644 --- a/pillar.example +++ b/pillar.example @@ -62,10 +62,10 @@ packages: - goodbye-world npms: - # dir: /home/kitchen/npms # The target directory in which to install the package, or None for global installation - # user: kitchen # The user to run NPM with (and to assign to `dir`) - # group: kitchen # The group to assign to `dir` - # mode: 0755 # The permissions to assign to `dir` + dir: /home/kitchen/npms # The target directory in which to install the package, or None for global installation + user: kitchen # The user to run NPM with (and to assign to `dir`) + group: kitchen # The group to assign to `dir` + mode: 0755 # The permissions to assign to `dir` # registry: None # The NPM registry from which to install the package # env: None # A list of environment variables to be set prior to execution # force_reinstall: False # Install the package even if it is already installed diff --git a/test/integration/default/npms_spec.rb b/test/integration/default/npms_spec.rb index debcb46..829aed1 100644 --- a/test/integration/default/npms_spec.rb +++ b/test/integration/default/npms_spec.rb @@ -1,3 +1,8 @@ +## FIXME! inspec's npm resource fails to check correctly (sudo issues, path issues) +## so I added some "poor man's checks" to ensure, at least, that npms are in place + +npms_path = '/home/kitchen/npms' + wanted_npms = { 'hello-world-npm': '1.1.1', 'sax': '1.2.4', @@ -13,12 +18,16 @@ control 'Wanted/Required npm packages' do os.name == 'debian' end + describe directory(npms_path) do + it { should exist } + its('owner') { should cmp 'kitchen' } + its('group') { should cmp 'kitchen' } + its('mode') { should cmp '0755' } + end + wanted_npms.each do |p,v| - describe npm(p) do - it { should be_installed } - # FIXME! Testing for version is failing, seems an issue in inspec - # Same happens with testing path, as it performs a cd command and it fails when using sudo - # its('version') { should eq v } + describe directory("#{npms_path}/node_modules/#{p}") do + it { should exist } end end end From 1904435e5f7f09e3aaf9d5fb94895d7d6de8fcc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20B=C3=A9rtoli?= Date: Mon, 11 Mar 2019 08:07:38 -0300 Subject: [PATCH 5/5] Add npm example starting with @ --- pillar.example | 2 ++ test/integration/default/npms_spec.rb | 1 + 2 files changed, 3 insertions(+) diff --git a/pillar.example b/pillar.example index 17d1b4b..681d04d 100644 --- a/pillar.example +++ b/pillar.example @@ -79,6 +79,8 @@ packages: # @foobar # buffer-equal-constant-time@1.0.1 # coffee-script + # You need to quote the package if it starts with '@' + - '@davidodio/hello@2.3.0' - hello-world-npm - sax - coffee-script@1.0.1 diff --git a/test/integration/default/npms_spec.rb b/test/integration/default/npms_spec.rb index 829aed1..d5f766b 100644 --- a/test/integration/default/npms_spec.rb +++ b/test/integration/default/npms_spec.rb @@ -4,6 +4,7 @@ npms_path = '/home/kitchen/npms' wanted_npms = { + '@davidodio/hello': '2.3.0', 'hello-world-npm': '1.1.1', 'sax': '1.2.4', 'coffee-script': '1.0.1'