Merge pull request #136 from vutny/start-postgresql-even-if-there-is-no-init

Attempt to start and manage PostgreSQL service even if init system fails

README.rst

@@ -16,8 +16,8 @@ Available states
 ``postgres``
 ------------
 
-Installs and configures both PostgreSQL server and client with creation of
+Installs and configures both PostgreSQL server and client with creation of various DB objects in
-various DB objects in the cluster.
+the cluster.
 
 ``postgres.client``
 -------------------
@@ -27,8 +27,8 @@ Installs the PostgreSQL client binaries and libraries.
 ``postgres.manage``
 -------------------
 
-Creates such DB objects as: users, tablespaces, databases, schemas and
+Creates such DB objects as: users, tablespaces, databases, schemas and extensions.
-extensions. See ``pillar.example`` file for details.
+See ``pillar.example`` file for details.
 
 ``postgres.python``
 -------------------
@@ -38,7 +38,34 @@ Installs the PostgreSQL adapter for Python.
 ``postgres.server``
 -------------------
 
-Installs the PostgreSQL server package and prepares the DB cluster.
+Installs the PostgreSQL server package, prepares the DB cluster and starts the server using
+packaged init script, job or unit.
+
+``postgres.server.image``
+-------------------------
+
+Installs the PostgreSQL server package, prepares the DB cluster and starts the server by issuing
+raw ``pg_ctl`` command. The ``postgres:bake_image`` Pillar toggles this behaviour. For example:
+
+.. code:: yaml
+
+  postgres:
+    bake_image: True
+
+If set ``True``, then it becomes possible to fully provision PostgreSQL with all supported entities
+from ``postgres.manage`` state during the build ("baking") of AMI / VM / Container images (using
+Packer, Docker or similar tools), i.e. when OS ``init`` process is not available to start the
+service and enable it on "boot" of resulting appliance.
+
+Also it allows to make Docker images with PostgreSQL using functionality being available since Salt
+2016.11.0 release:
+
+.. code:: console
+
+  salt 'minion.with.docker' dockerng.sls_build my-postgres base=centos/systemd mods=postgres
+
+If a lookup dictionary or Pillar has ``postgres:bake_image`` set ``False`` (this is default), it is
+equivalent of applying ``postgres.server`` state.
 
 ``postgres.upstream``
 ---------------------
@@ -46,14 +73,13 @@ Installs the PostgreSQL server package and prepares the DB cluster.
 Configures the PostgreSQL Official (upstream) repository on target system if
 applicable.
 
-The state relies on the ``postgres:use_upstream_repo`` Pillar value which could
+The state relies on the ``postgres:use_upstream_repo`` Pillar value which could be set as following:
-be set as following:
 
 * ``True`` (default): adds the upstream repository to install packages from
 * ``False``: makes sure that the repository configuration is absent
 
-The ``postgres:version`` Pillar controls which version of the PostgreSQL
+The ``postgres:version`` Pillar controls which version of the PostgreSQL packages should be
-packages should be installed from the upstream repository. Defaults to ``9.5``.
+installed from the upstream repository. Defaults to ``9.5``.
 
 Testing
 =======
@@ -79,3 +105,5 @@ Builds and runs tests from scratch.
 -----------------
 
 Gives you ssh to the vagrant machine for manual testing.
+
+.. vim: fenc=utf-8 spell spl=en cc=100 tw=99 fo=want sts=2 sw=2 et

pillar.example

@@ -19,7 +19,7 @@ postgres:
   postgresconf: |
     listen_addresses = '*'  # listen on all interfaces
 
-  # Backup extension for postgresql.conf file, defaults to `.bak`.
+  # Backup extension for postgresql.conf file, defaults to ``.bak``.
   # Set to False to stop creation of backup on postgresql.conf changes.
   {%- if 'status.time' in salt.keys() %}
   postgresconf_backup: ".backup@{{ salt['status.time']('%y-%m-%d_%H:%M:%S') }}"
@@ -28,7 +28,7 @@ postgres:
   # Path to the `pg_hba.conf` file Jinja template on Salt Fileserver
   pg_hba.conf: salt://postgres/templates/pg_hba.conf.j2
 
-  # This section covers ACL management in the `pg_hba.conf` file.
+  # This section covers ACL management in the ``pg_hba.conf`` file.
   # acls list controls: which hosts are allowed to connect, how clients
   # are authenticated, which PostgreSQL user names they can use, which
   # databases they can access. Records take one of these forms:
@@ -48,9 +48,20 @@ postgres:
   # PostgreSQL service name
   service: postgresql
 
+  {%- if grains['init'] == 'unknown' %}
+
+  # If Salt is unable to detect init system running in the scope of state run,
+  # probably we are trying to bake a container/VM image with PostgreSQL.
+  # Use ``bake_image`` setting to contol how PostgreSQL will be started: if set
+  # to ``True`` the raw ``pg_ctl`` will be utilized instead of packaged init
+  # script, job or unit run with Salt ``service`` state.
+  bake_image: True
+
+  {%- endif %}
+
   # Create/remove users, tablespaces, databases, schema and extensions.
   # Each of these dictionaries contains PostgreSQL entities which
-  # mapped to the `postgres_*` Salt states with arguments. See the Salt
+  # mapped to the ``postgres_*`` Salt states with arguments. See the Salt
   # documentaion to get all supported argument for a particular state.
   #
   # Format is the following:

postgres/codenamemap.yaml

@@ -31,7 +31,6 @@
     command: pg_createcluster {{ version }} main
     test: test -f /var/lib/postgresql/{{ version }}/main/PG_VERSION && test -f /etc/postgresql/{{ version }}/main/postgresql.conf
     user: root
-    env: {}
 
 {% endmacro %}
 

postgres/defaults.yaml

@@ -12,15 +12,10 @@ postgres:
   user: postgres
   group: postgres
 
-  # if prepare_cluster is over-ridden in any of:
-  # - osmap.yaml
-  # - oscodenamemap.yaml
-  # - osfingermap.yaml
-  # you will have to specify a complete dictionary.
   prepare_cluster:
-    user: root
+    command: initdb --pgdata=/var/lib/pgsql/data
-    command: service postgresql initdb
     test: test -f /var/lib/pgsql/data/PG_VERSION
+    user: postgres
     env: {}
 
   conf_dir: /var/lib/pgsql/data
@@ -38,6 +33,8 @@ postgres:
 
   service: postgresql
 
+  bake_image: False
+
   users: {}
   tablespaces: {}
   databases: {}

postgres/map.jinja

@@ -1,7 +1,6 @@
 {% import_yaml "postgres/defaults.yaml" as defaults %}
 {% import_yaml "postgres/osmap.yaml" as osmap %}
 {% import_yaml "postgres/codenamemap.yaml" as oscodenamemap %}
-{% import_yaml "postgres/osmajorreleasemap.yaml" as osmajorreleasemap %}
 
 {% set postgres = salt['grains.filter_by'](
     defaults,
@@ -11,12 +10,8 @@
         merge=salt['grains.filter_by'](
             oscodenamemap,
             grain='oscodename',
-            merge=salt['grains.filter_by'](
-                osmajorreleasemap,
-                grain='osmajorrelease',
             merge=salt['pillar.get']('postgres', {}),
         ),
     ),
-    ),
     base='postgres',
 ) %}

postgres/osmajorreleasemap.yaml

@@ -1,67 +0,0 @@
-{% import_yaml "postgres/repo.yaml" as repo %}
-
-{% if grains['os_family'] == 'RedHat' %}
-
-### RedHat releases
-
-  {% if repo.use_upstream_repo %}
-
-    {% set data_dir = '/var/lib/pgsql/' ~ repo.version ~ '/data' %}
-
-# PostgreSQL from upstream repository
-
-default:
-  prepare_cluster:
-    user: postgres
-    command: /usr/pgsql-{{ repo.version }}/bin/initdb -D {{ data_dir }}
-    test: test -f {{ data_dir }}/PG_VERSION
-    env: {}
-'6':
-  prepare_cluster:
-    user: root
-    command: service postgresql-{{ repo.version }} initdb
-    test: test -f {{ data_dir }}/PG_VERSION
-    env: {}
-'7':
-  prepare_cluster:
-    user: root
-    command: /usr/pgsql-{{ repo.version }}/bin/postgresql95-setup initdb
-    test: test -f {{ data_dir }}/PG_VERSION
-    env: {}
-
-  {% else %}
-
-    {% set data_dir = '/var/lib/pgsql/data' %}
-
-# PostgreSQL from OS repositories
-
-default:
-  prepare_cluster:
-    user: postgres
-    command: initdb -D {{ data_dir }}
-    test: test -f {{ data_dir }}/PG_VERSION
-    env: {}
-'6':
-  prepare_cluster:
-    user: root
-    command: service postgresql initdb
-    test: test -f {{ data_dir }}/PG_VERSION
-    env: {}
-'7':
-  prepare_cluster:
-    user: root
-    command: postgresql-setup initdb
-    test: test -f {{ data_dir }}/PG_VERSION
-    env: {}
-
-  {% endif %}
-
-{% else %}
-
-### Empty YAML, no settings for other OS
-
-{}
-
-{% endif %}
-
-# vim: ft=sls

postgres/osmap.yaml

@@ -5,10 +5,8 @@
 Arch:
   conf_dir: /var/lib/postgres/data
   prepare_cluster:
-    user: postgres
     command: initdb -D /var/lib/postgresql/data
     test: test -f /var/lib/postgres/data/PG_VERSION
-    env: {}
   pkg_client: postgresql
   pkg_dev: postgresql
 
@@ -37,11 +35,17 @@ RedHat:
 
 {% if repo.use_upstream_repo %}
 
+  {% set data_dir = '/var/lib/pgsql/' ~ repo.version ~ '/data' %}
+
   pkg: postgresql{{ release }}-server
   pkg_client: postgresql{{ release }}
   conf_dir: /var/lib/pgsql/{{ repo.version }}/data
   service: postgresql-{{ repo.version }}
 
+  prepare_cluster:
+    command: initdb --pgdata='{{ data_dir }}'
+    test: test -f '{{ data_dir }}/PG_VERSION'
+
   # Directory containing PostgreSQL client executables
   bin_dir: /usr/pgsql-{{ repo.version }}/bin
   client_bins:

postgres/server/image.sls

@@ -0,0 +1,45 @@
+{%- from "postgres/map.jinja" import postgres with context -%}
+
+# This state is used to launch PostgreSQL with ``pg_ctl`` command and enable it
+# on "boot" during an image (Docker, Virtual Appliance, AMI) preparation
+
+{%- if postgres.bake_image %}
+
+include:
+  - postgres.server
+
+# An attempt to start PostgreSQL with `pg_ctl`
+
+postgresql-start:
+  cmd.run:
+    - name: pg_ctl -D {{ postgres.conf_dir }} -l logfile start
+    - runas: {{ postgres.user }}
+    - unless:
+      - ps -p $(head -n 1 {{ postgres.conf_dir }}/postmaster.pid) 2>/dev/null
+    - require:
+      - file: postgresql-pg_hba
+
+# Try to enable PostgreSQL in "manual" way
+
+postgresql-enable:
+  cmd.run:
+  {%- if salt['file.file_exists']('/bin/systemctl') %}
+    - name: systemctl enable {{ postgres.service }}
+  {%- elif salt['cmd.which']('chkconfig') %}
+    - name: chkconfig {{ postgres.service }} on
+  {%- elif salt['file.file_exists']('/usr/sbin/update-rc.d') %}
+    - name: update-rc.d {{ service }} defaults
+  {%- else %}
+    # Nothing to do
+    - name: 'true'
+  {%- endif %}
+    - require:
+      - cmd: postgresql-start
+
+{%- else %}
+
+postgresql-start:
+  test.show_notification:
+    - text: The 'postgres:bake_image' Pillar is disabled (set to 'False').
+
+{%- endif %}

postgres/server/init.sls

@@ -1,11 +1,19 @@
-{%- from "postgres/map.jinja" import postgres with context -%}
+{%- from "postgres/map.jinja" import postgres with context %}
-
-{%- set pkgs = [postgres.pkg] + postgres.pkgs_extra -%}
 
+{%- set includes = [] %}
+{%- if postgres.bake_image %}
+  {%- do includes.append('postgres.server.image') %}
+{%- endif %}
 {%- if postgres.use_upstream_repo -%}
+  {%- do includes.append('postgres.upstream') %}
+{%- endif %}
+
+{%- set pkgs = [postgres.pkg] + postgres.pkgs_extra %}
+
+{%- if includes -%}
 
 include:
-  - postgres.upstream
+  {{ includes|yaml(false)|indent(2) }}
 
 {%- endif %}
 
@@ -20,6 +28,29 @@ postgresql-server:
       - pkgrepo: postgresql-repo
 {%- endif %}
 
+{%- if 'bin_dir' in postgres %}
+
+# Make server binaries available in $PATH
+
+  {%- for bin in postgres.server_bins %}
+
+    {%- set path = salt['file.join'](postgres.bin_dir, bin) %}
+
+{{ bin }}:
+  alternatives.install:
+    - link: {{ salt['file.join']('/usr/bin', bin) }}
+    - path: {{ path }}
+    - priority: 30
+    - onlyif: test -f {{ path }}
+    - require:
+      - pkg: postgresql-server
+    - require_in:
+      - cmd: postgresql-cluster-prepared
+
+  {%- endfor %}
+
+{%- endif %}
+
 postgresql-cluster-prepared:
   cmd.run:
     - name: {{ postgres.prepare_cluster.command }}
@@ -67,17 +98,11 @@ postgresql-pg_hba:
     - user: {{ postgres.user }}
     - group: {{ postgres.group }}
     - mode: 600
+    - defaults:
+        acls: {{ postgres.acls }}
     - require:
       - file: postgresql-config-dir
 
-postgresql-running:
-  service.running:
-    - name: {{ postgres.service }}
-    - enable: True
-    - reload: True
-    - watch:
-      - file: postgresql-pg_hba
-
 {%- for name, tblspace in postgres.tablespaces|dictsort() %}
 
 postgresql-tablespace-dir-{{ name }}:
@@ -90,26 +115,21 @@ postgresql-tablespace-dir-{{ name }}:
     - recurse:
       - user
       - group
-
-{%- endfor %}
-
-{%- if 'bin_dir' in postgres %}
-
-# Make server binaries available in $PATH
-
-  {%- for bin in postgres.server_bins %}
-
-    {%- set path = salt['file.join'](postgres.bin_dir, bin) %}
-
-{{ bin }}:
-  alternatives.install:
-    - link: {{ salt['file.join']('/usr/bin', bin) }}
-    - path: {{ path }}
-    - priority: 30
-    - onlyif: test -f {{ path }}
     - require:
       - pkg: postgresql-server
 
-  {%- endfor %}
+{%- endfor %}
+
+{%- if not postgres.bake_image %}
+
+# Start PostgreSQL server using OS init
+
+postgresql-running:
+  service.running:
+    - name: {{ postgres.service }}
+    - enable: True
+    - reload: True
+    - watch:
+      - file: postgresql-pg_hba
 
 {%- endif %}

postgres/templates/pg_hba.conf.j2

@@ -1,5 +1,3 @@
-{%- from "postgres/map.jinja" import postgres with context -%}
-
 ######################################################################
 # ATTENTION! Managed by SaltStack.                                   #
 # DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN! #
@@ -22,7 +20,7 @@ local   all             postgres                                peer
 
 # TYPE  DATABASE        USER            ADDRESS                 METHOD
 
-{%- for acl in postgres.acls %}
+{%- for acl in acls %}
   {%- if acl|first() == 'local' %}
 
     {%- if acl|length() == 3 %}