From d5dcac95a7cf77ee028b2ddc4874a5495f3baea5 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Mon, 26 Sep 2016 13:57:55 +0300 Subject: [PATCH 01/42] RedHat: make client binaries available in `$PATH` --- postgres/client.sls | 52 ++++++++++++++++++++++++++++++------------ postgres/defaults.yaml | 27 ++++++++++++++++++++++ postgres/init.sls | 10 ++++---- postgres/osmap.yaml | 1 + postgres/upstream.sls | 6 ++--- 5 files changed, 74 insertions(+), 22 deletions(-) diff --git a/postgres/client.sls b/postgres/client.sls index 95d06b8..bf11dec 100644 --- a/postgres/client.sls +++ b/postgres/client.sls @@ -1,21 +1,45 @@ -{% from "postgres/map.jinja" import postgres with context %} +{%- from "postgres/map.jinja" import postgres with context -%} + +{%- set pkgs = [] %} +{%- for pkg in (postgres.pkg_client, postgres.pkg_libpq_dev) %} + {%- if pkg %} + {%- do pkgs.append(pkg) %} + {%- endif %} +{%- endfor -%} + +{%- if postgres.use_upstream_repo %} -{% if postgres.use_upstream_repo %} include: - postgres.upstream -{% endif %} -install-postgresql-client: - pkg.installed: - - name: {{ postgres.pkg_client }} - - refresh: {{ postgres.use_upstream_repo }} -{% if postgres.use_upstream_repo %} - - require: - - pkgrepo: install-postgresql-repo {%- endif %} -{% if postgres.pkg_libpq_dev %} -install-postgres-libpq-dev: +postgresql-client-libs: pkg.installed: - - name: {{ postgres.pkg_libpq_dev }} -{% endif %} + - pkgs: {{ pkgs }} +{%- if postgres.use_upstream_repo %} + - refresh: True + - require: + - pkgrepo: postgresql-repo +{%- endif %} + +{%- if 'bin_dir' in postgres %} + +# Make client binaries available in $PATH + + {%- for bin in postgres.client_bins %} + + {%- set path = salt['file.join'](postgres.bin_dir, bin) %} + +{{ bin }}: + alternatives.install: + - link: {{ salt['file.join']('/usr/bin', bin) }} + - path: {{ path }} + - priority: 30 + - onlyif: test -f {{ path }} + - require: + - pkg: postgresql-client-libs + + {%- endfor %} + +{%- endif %} diff --git a/postgres/defaults.yaml b/postgres/defaults.yaml index 9172a9b..805fed5 100644 --- a/postgres/defaults.yaml +++ b/postgres/defaults.yaml @@ -39,3 +39,30 @@ postgres: command: service postgresql initdb test: test -f /var/lib/pgsql/data/PG_VERSION env: {} + + # Client executables list + client_bins: + - clusterdb + - createdb + - createlang + - createuser + - dropdb + - droplang + - dropuser + - pg_archivecleanup + - pg_basebackup + - pg_config + - pg_dump + - pg_dumpall + - pg_isready + - pg_receivexlog + - pg_restore + - pg_rewind + - pg_test_fsync + - pg_test_timing + - pg_upgrade + - pg_xlogdump + - pgbench + - psql + - reindexdb + - vacuumdb diff --git a/postgres/init.sls b/postgres/init.sls index bce0647..548b979 100644 --- a/postgres/init.sls +++ b/postgres/init.sls @@ -12,13 +12,13 @@ include: ### Installation states -postgresql-installed: +postgresql-server: pkg.installed: - name: {{ postgres.pkg }} - - refresh: {{ postgres.use_upstream_repo }} -{% if postgres.use_upstream_repo %} +{%- if postgres.use_upstream_repo %} + - refresh: True - require: - - pkgrepo: install-postgresql-repo + - pkgrepo: postgresql-repo {%- endif %} # make sure the data directory and contents have been initialized @@ -31,7 +31,7 @@ postgresql-cluster-prepared: - unless: - {{ postgres.prepare_cluster.test }} - require: - - pkg: postgresql-installed + - pkg: postgresql-server postgresql-config-dir: file.directory: diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 23a1085..7864a00 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -39,6 +39,7 @@ RedHat: pkg: postgresql{{ release }}-server pkg_client: postgresql{{ release }} + bin_dir: /usr/pgsql-{{ repo.version }}/bin conf_dir: /var/lib/pgsql/{{ repo.version }}/data service: postgresql-{{ repo.version }} diff --git a/postgres/upstream.sls b/postgres/upstream.sls index 1d5c886..6c3956f 100644 --- a/postgres/upstream.sls +++ b/postgres/upstream.sls @@ -6,14 +6,14 @@ {%- if postgres.use_upstream_repo -%} # Add upstream repository for your distro -install-postgresql-repo: +postgresql-repo: pkgrepo.managed: {{- format_kwargs(postgres.pkg_repo) }} {%- else -%} # Remove the repo configuration (and GnuPG key) as requested -remove-postgresql-repo: +postgresql-repo: pkgrepo.absent: - name: {{ postgres.pkg_repo.name }} {%- if 'pkg_repo_keyid' in postgres %} @@ -25,7 +25,7 @@ remove-postgresql-repo: {%- else -%} # Notify that we don't manage this distro -install-postgresql-repo: +postgresql-repo: test.show_notification: - text: | PostgreSQL does not provide package repository for {{ grains['osfinger'] }} From 95187374778bf9a7d7e2e75f36f22b5aa8f44b51 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Mon, 3 Oct 2016 14:04:26 +0300 Subject: [PATCH 02/42] Move setting `client_bins` item to the `osmap` lookup dict --- postgres/defaults.yaml | 27 --------------------------- postgres/osmap.yaml | 29 ++++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/postgres/defaults.yaml b/postgres/defaults.yaml index 805fed5..9172a9b 100644 --- a/postgres/defaults.yaml +++ b/postgres/defaults.yaml @@ -39,30 +39,3 @@ postgres: command: service postgresql initdb test: test -f /var/lib/pgsql/data/PG_VERSION env: {} - - # Client executables list - client_bins: - - clusterdb - - createdb - - createlang - - createuser - - dropdb - - droplang - - dropuser - - pg_archivecleanup - - pg_basebackup - - pg_config - - pg_dump - - pg_dumpall - - pg_isready - - pg_receivexlog - - pg_restore - - pg_rewind - - pg_test_fsync - - pg_test_timing - - pg_upgrade - - pg_xlogdump - - pgbench - - psql - - reindexdb - - vacuumdb diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 7864a00..ebef733 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -39,10 +39,37 @@ RedHat: pkg: postgresql{{ release }}-server pkg_client: postgresql{{ release }} - bin_dir: /usr/pgsql-{{ repo.version }}/bin conf_dir: /var/lib/pgsql/{{ repo.version }}/data service: postgresql-{{ repo.version }} + # Directory containing PostgreSQL client executables + bin_dir: /usr/pgsql-{{ repo.version }}/bin + client_bins: + - clusterdb + - createdb + - createlang + - createuser + - dropdb + - droplang + - dropuser + - pg_archivecleanup + - pg_basebackup + - pg_config + - pg_dump + - pg_dumpall + - pg_isready + - pg_receivexlog + - pg_restore + - pg_rewind + - pg_test_fsync + - pg_test_timing + - pg_upgrade + - pg_xlogdump + - pgbench + - psql + - reindexdb + - vacuumdb + {% else %} pkg: postgresql-server From 234a76f1e9cf76f6fa655e7823b1f0b7d13b4dcd Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Tue, 4 Oct 2016 14:38:21 +0300 Subject: [PATCH 03/42] Manage PostgreSQL entities with `client.sls` states --- README.rst | 10 ++- pillar.example | 3 +- postgres/client.sls | 80 ++++++++++++++++++- postgres/init.sls | 182 +------------------------------------------- postgres/server.sls | 94 +++++++++++++++++++++++ 5 files changed, 186 insertions(+), 183 deletions(-) create mode 100644 postgres/server.sls diff --git a/README.rst b/README.rst index c77adee..d6a3040 100644 --- a/README.rst +++ b/README.rst @@ -16,18 +16,26 @@ Available states ``postgres`` ------------ -Installs the PostgreSQL server package and prepares the DB cluster. +Installs and configures both PostgreSQL server and client with creation of +various DB objects in the cluster. ``postgres.client`` ------------------- Installs the PostgreSQL client binaries and libraries. +Allows to create such DB objects as: users, tablespaces, databases, schemas and +extensions. See ``pillar.example`` file for details. ``postgres.python`` ------------------- Installs the PostgreSQL adapter for Python. +``postgres.server`` +------------------- + +Installs the PostgreSQL server package and prepares the DB cluster. + ``postgres.upstream`` --------------------- diff --git a/pillar.example b/pillar.example index 82c357d..13e9887 100644 --- a/pillar.example +++ b/pillar.example @@ -8,7 +8,8 @@ postgres: pkg: 'postgresql-9.3' pkg_client: 'postgresql-client-9.3' - # Addtional packages to install, this should be in a list format + # Additional packages to install with PostgreSQL server, + # this should be in a list format pkgs_extra: - postgresql-contrib - postgresql-plpython diff --git a/postgres/client.sls b/postgres/client.sls index bf11dec..210548c 100644 --- a/postgres/client.sls +++ b/postgres/client.sls @@ -1,4 +1,5 @@ {%- from "postgres/map.jinja" import postgres with context -%} +{%- from "postgres/macros.jinja" import format_state with context -%} {%- set pkgs = [] %} {%- for pkg in (postgres.pkg_client, postgres.pkg_libpq_dev) %} @@ -7,13 +8,15 @@ {%- endif %} {%- endfor -%} -{%- if postgres.use_upstream_repo %} +{%- if postgres.use_upstream_repo -%} include: - postgres.upstream {%- endif %} +# Install PostgreSQL client and libraries + postgresql-client-libs: pkg.installed: - pkgs: {{ pkgs }} @@ -43,3 +46,78 @@ postgresql-client-libs: {%- endfor %} {%- endif %} + +# Ensure that Salt is able to use postgres modules +# after installing client binaries + +postgres-reload-modules: + test.nop: + - reload_modules: True + +# User states + +{%- for name, user in postgres.users|dictsort() %} + +{{ format_state(name, 'postgres_user', user) }} + - require: + - pkg: postgresql-client-libs + +{%- endfor %} + +# Tablespace states + +{%- for name, tblspace in postgres.tablespaces|dictsort() %} + +{{ format_state(name, 'postgres_tablespace', tblspace) }} + - require: + - pkg: postgresql-client-libs + {%- if 'owner' in tblspace %} + - postgres_user: postgres_user-{{ tblspace.owner }} + {%- endif %} + +{%- endfor %} + +# Database states + +{%- for name, db in postgres.databases|dictsort() %} + +{{ format_state(name, 'postgres_database', db) }} + - require: + - pkg: postgresql-client-libs + {%- if 'owner' in db %} + - postgres_user: postgres_user-{{ db.owner }} + {%- endif %} + {%- if 'tablespace' in db %} + - postgres_tablespace: postgres_tablespace-{{ db.tablespace }} + {%- endif %} + +{%- endfor %} + +# Schema states + +{%- for name, schema in postgres.schemas|dictsort() %} + +{{ format_state(name, 'postgres_schema', schema) }} + - require: + - pkg: postgresql-client-libs + {%- if 'owner' in schema %} + - postgres_user: postgres_user-{{ schema.owner }} + {%- endif %} + +{%- endfor %} + +# Extension states + +{%- for name, extension in postgres.extensions|dictsort() %} + +{{ format_state(name, 'postgres_extension', extension) }} + - require: + - pkg: postgresql-client-libs + {%- if 'maintenance_db' in extension %} + - postgres_database: postgres_database-{{ extension.maintenance_db }} + {%- endif %} + {%- if 'schema' in extension %} + - postgres_schema: postgres_schema-{{ extension.schema }} + {%- endif %} + +{%- endfor %} diff --git a/postgres/init.sls b/postgres/init.sls index 548b979..0dc4299 100644 --- a/postgres/init.sls +++ b/postgres/init.sls @@ -1,181 +1,3 @@ -# -*- mode: yaml -*- - -{%- from "postgres/map.jinja" import postgres with context -%} -{%- from "postgres/macros.jinja" import format_state with context -%} - -{%- if postgres.use_upstream_repo %} - include: - - postgres.upstream - -{%- endif %} - -### Installation states - -postgresql-server: - pkg.installed: - - name: {{ postgres.pkg }} -{%- if postgres.use_upstream_repo %} - - refresh: True - - require: - - pkgrepo: postgresql-repo -{%- endif %} - -# make sure the data directory and contents have been initialized -postgresql-cluster-prepared: - cmd.run: - - name: {{ postgres.prepare_cluster.command }} - - cwd: / - - runas: {{ postgres.prepare_cluster.user }} - - env: {{ postgres.prepare_cluster.env|default({}) }} - - unless: - - {{ postgres.prepare_cluster.test }} - - require: - - pkg: postgresql-server - -postgresql-config-dir: - file.directory: - - name: {{ postgres.conf_dir }} - - user: {{ postgres.user }} - - group: {{ postgres.group }} - - makedirs: True - - require: - - cmd: postgresql-cluster-prepared - -{%- if postgres.postgresconf %} - -postgresql-conf: - file.blockreplace: - - name: {{ postgres.conf_dir }}/postgresql.conf - - marker_start: "# Managed by SaltStack: listen_addresses: please do not edit" - - marker_end: "# Managed by SaltStack: end of salt managed zone --" - - content: | - {{ postgres.postgresconf|indent(8) }} - - show_changes: True - - append_if_not_found: True - - backup: {{ postgres.postgresconf_backup }} - - watch_in: - - service: postgresql-running - - require: - - file: postgresql-config-dir - -{%- endif %} - -postgresql-pg_hba: - file.managed: - - name: {{ postgres.conf_dir }}/pg_hba.conf - - source: {{ postgres['pg_hba.conf'] }} - - template: jinja - - user: {{ postgres.user }} - - group: {{ postgres.group }} - - mode: 600 - - require: - - file: postgresql-config-dir - -postgresql-running: - service.running: - - name: {{ postgres.service }} - - enable: True - - reload: True - - watch: - - file: postgresql-pg_hba - -postgresql-extra-pkgs-installed: - pkg.installed: - - pkgs: {{ postgres.pkgs_extra }} - -### User states - -{%- for name, user in postgres.users|dictsort() %} - -{{ format_state(name, 'postgres_user', user) }} - - require: - - service: postgresql-running - {%- if 'db_user' in user %} - - postgres_user: postgres_user-{{ user.db_user }} - {%- endif %} - -{%- endfor %} - -### Tablespace states - -{%- for name, tblspace in postgres.tablespaces|dictsort() %} - -postgres_tablespace-dir-{{ tblspace.directory}}: - file.directory: - - name: {{ tblspace.directory }} - - user: {{ postgres.user }} - - group: {{ postgres.group }} - - mode: 700 - - makedirs: True - - recurse: - - user - - group - -{{ format_state(name, 'postgres_tablespace', tblspace) }} - - require: - - file: postgres_tablespace-dir-{{ tblspace.directory }} - {%- if 'owner' in tblspace %} - - postgres_user: postgres_user-{{ tblspace.owner }} - {%- endif %} - - service: postgresql-running - -{%- endfor %} - -### Database states - -{%- for name, db in postgres.databases|dictsort() %} - -{{ format_state(name, 'postgres_database', db) }} - - require: - - service: postgresql-running - {%- if 'db_user' in db %} - - postgres_user: postgres_user-{{ db.db_user }} - {%- endif %} - {%- if 'owner' in db %} - - postgres_user: postgres_user-{{ db.owner }} - {%- endif %} - {%- if 'tablespace' in db %} - - postgres_tablespace: postgres_tablespace-{{ db.tablespace }} - {%- endif %} - -{%- endfor %} - -### Schema states - -{%- for name, schema in postgres.schemas|dictsort() %} - -{{ format_state(name, 'postgres_schema', schema) }} - - require: - - service: postgresql-running - {%- if 'db_user' in schema %} - - postgres_user: postgres_user-{{ schema.db_user }} - {%- endif %} - {%- if 'dbname' in schema %} - - postgres_database: postgres_database-{{ schema.dbname }} - {%- endif %} - {%- if 'owner' in schema %} - - postgres_user: postgres_user-{{ schema.owner }} - {%- endif %} - -{%- endfor %} - -### Extension states - -{%- for name, extension in postgres.extensions|dictsort() %} - -{{ format_state(name, 'postgres_extension', extension) }} - - require: - - service: postgresql-running - - pkg: postgresql-extra-pkgs-installed - {%- if 'db_user' in extension %} - - postgres_user: postgres_user-{{ extension.db_user }} - {%- endif %} - {%- if 'maintenance_db' in extension %} - - postgres_database: postgres_database-{{ extension.maintenance_db }} - {%- endif %} - {%- if 'schema' in extension %} - - postgres_schema: postgres_schema-{{ extension.schema }} - {%- endif %} - -{%- endfor %} + - postgres.server + - postgres.client diff --git a/postgres/server.sls b/postgres/server.sls new file mode 100644 index 0000000..5efb04f --- /dev/null +++ b/postgres/server.sls @@ -0,0 +1,94 @@ +{%- from "postgres/map.jinja" import postgres with context -%} + +{%- set pkgs = [postgres.pkg] + postgres.pkgs_extra -%} + +{%- if postgres.use_upstream_repo -%} + +include: + - postgres.upstream + +{%- endif %} + +# Install, configure and start PostgreSQL server + +postgresql-server: + pkg.installed: + - pkgs: {{ pkgs }} +{%- if postgres.use_upstream_repo %} + - refresh: True + - require: + - pkgrepo: postgresql-repo +{%- endif %} + +postgresql-cluster-prepared: + cmd.run: + - name: {{ postgres.prepare_cluster.command }} + - cwd: / + - runas: {{ postgres.prepare_cluster.user }} + - env: {{ postgres.prepare_cluster.env|default({}) }} + - unless: + - {{ postgres.prepare_cluster.test }} + - require: + - pkg: postgresql-server + +postgresql-config-dir: + file.directory: + - name: {{ postgres.conf_dir }} + - user: {{ postgres.user }} + - group: {{ postgres.group }} + - makedirs: True + - require: + - cmd: postgresql-cluster-prepared + +{%- if postgres.postgresconf %} + +postgresql-conf: + file.blockreplace: + - name: {{ postgres.conf_dir }}/postgresql.conf + - marker_start: "# Managed by SaltStack: listen_addresses: please do not edit" + - marker_end: "# Managed by SaltStack: end of salt managed zone --" + - content: | + {{ postgres.postgresconf|indent(8) }} + - show_changes: True + - append_if_not_found: True + - backup: {{ postgres.postgresconf_backup }} + - require: + - file: postgresql-config-dir + - watch_in: + - service: postgresql-running + +{%- endif %} + +postgresql-pg_hba: + file.managed: + - name: {{ postgres.conf_dir }}/pg_hba.conf + - source: {{ postgres['pg_hba.conf'] }} + - template: jinja + - user: {{ postgres.user }} + - group: {{ postgres.group }} + - mode: 600 + - require: + - file: postgresql-config-dir + +postgresql-running: + service.running: + - name: {{ postgres.service }} + - enable: True + - reload: True + - watch: + - file: postgresql-pg_hba + +{%- for name, tblspace in postgres.tablespaces|dictsort() %} + +postgresql-tablespace-dir-{{ name }}: + file.directory: + - name: {{ tblspace.directory }} + - user: {{ postgres.user }} + - group: {{ postgres.group }} + - mode: 700 + - makedirs: True + - recurse: + - user + - group + +{%- endfor %} From a04fe4681b61fb77b3ad37acb24384e45abd8c72 Mon Sep 17 00:00:00 2001 From: Marc-Alexandre Chan Date: Sun, 9 Oct 2016 07:29:52 -0400 Subject: [PATCH 04/42] Fix missing spaces in long pg_hba.conf lines In generated pg_hba.conf, fix missing spaces between arguments if an argument is longer than the column width allotted to that argument in the line. --- postgres/templates/pg_hba.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/templates/pg_hba.conf.j2 b/postgres/templates/pg_hba.conf.j2 index 06a8af1..560c45e 100644 --- a/postgres/templates/pg_hba.conf.j2 +++ b/postgres/templates/pg_hba.conf.j2 @@ -38,5 +38,5 @@ local all postgres peer {%- endif %} {%- endif %} -{{ '{0:<8}{1:<16}{2:<16}{3:<24}{4}'.format(*acl) -}} +{{ '{0:<7} {1:<15} {2:<15} {3:<23} {4}'.format(*acl) -}} {% endfor %} From 74ddea8bdbceafff20c5b469412cd5e7f68e3f44 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Thu, 20 Oct 2016 12:23:40 +0300 Subject: [PATCH 05/42] [REFACTORING] Add `postgres.manage` state to provision DB objects --- README.rst | 6 +++- postgres/client.sls | 80 ++--------------------------------------- postgres/init.sls | 1 + postgres/manage.sls | 86 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 79 deletions(-) create mode 100644 postgres/manage.sls diff --git a/README.rst b/README.rst index d6a3040..b8d547e 100644 --- a/README.rst +++ b/README.rst @@ -23,7 +23,11 @@ various DB objects in the cluster. ------------------- Installs the PostgreSQL client binaries and libraries. -Allows to create such DB objects as: users, tablespaces, databases, schemas and + +``postgres.manage`` +------------------- + +Creates such DB objects as: users, tablespaces, databases, schemas and extensions. See ``pillar.example`` file for details. ``postgres.python`` diff --git a/postgres/client.sls b/postgres/client.sls index 210548c..c51ca11 100644 --- a/postgres/client.sls +++ b/postgres/client.sls @@ -1,14 +1,13 @@ {%- from "postgres/map.jinja" import postgres with context -%} -{%- from "postgres/macros.jinja" import format_state with context -%} {%- set pkgs = [] %} {%- for pkg in (postgres.pkg_client, postgres.pkg_libpq_dev) %} {%- if pkg %} {%- do pkgs.append(pkg) %} {%- endif %} -{%- endfor -%} +{%- endfor %} -{%- if postgres.use_upstream_repo -%} +{%- if postgres.use_upstream_repo %} include: - postgres.upstream @@ -46,78 +45,3 @@ postgresql-client-libs: {%- endfor %} {%- endif %} - -# Ensure that Salt is able to use postgres modules -# after installing client binaries - -postgres-reload-modules: - test.nop: - - reload_modules: True - -# User states - -{%- for name, user in postgres.users|dictsort() %} - -{{ format_state(name, 'postgres_user', user) }} - - require: - - pkg: postgresql-client-libs - -{%- endfor %} - -# Tablespace states - -{%- for name, tblspace in postgres.tablespaces|dictsort() %} - -{{ format_state(name, 'postgres_tablespace', tblspace) }} - - require: - - pkg: postgresql-client-libs - {%- if 'owner' in tblspace %} - - postgres_user: postgres_user-{{ tblspace.owner }} - {%- endif %} - -{%- endfor %} - -# Database states - -{%- for name, db in postgres.databases|dictsort() %} - -{{ format_state(name, 'postgres_database', db) }} - - require: - - pkg: postgresql-client-libs - {%- if 'owner' in db %} - - postgres_user: postgres_user-{{ db.owner }} - {%- endif %} - {%- if 'tablespace' in db %} - - postgres_tablespace: postgres_tablespace-{{ db.tablespace }} - {%- endif %} - -{%- endfor %} - -# Schema states - -{%- for name, schema in postgres.schemas|dictsort() %} - -{{ format_state(name, 'postgres_schema', schema) }} - - require: - - pkg: postgresql-client-libs - {%- if 'owner' in schema %} - - postgres_user: postgres_user-{{ schema.owner }} - {%- endif %} - -{%- endfor %} - -# Extension states - -{%- for name, extension in postgres.extensions|dictsort() %} - -{{ format_state(name, 'postgres_extension', extension) }} - - require: - - pkg: postgresql-client-libs - {%- if 'maintenance_db' in extension %} - - postgres_database: postgres_database-{{ extension.maintenance_db }} - {%- endif %} - {%- if 'schema' in extension %} - - postgres_schema: postgres_schema-{{ extension.schema }} - {%- endif %} - -{%- endfor %} diff --git a/postgres/init.sls b/postgres/init.sls index 0dc4299..0409e62 100644 --- a/postgres/init.sls +++ b/postgres/init.sls @@ -1,3 +1,4 @@ include: - postgres.server - postgres.client + - postgres.manage diff --git a/postgres/manage.sls b/postgres/manage.sls new file mode 100644 index 0000000..ea31836 --- /dev/null +++ b/postgres/manage.sls @@ -0,0 +1,86 @@ +{%- from "postgres/map.jinja" import postgres with context -%} +{%- from "postgres/macros.jinja" import format_state with context -%} + +{%- if not salt.get('postgres.user_create') %} + +# Salt states for managing PostgreSQL is not available, +# need to provision client binaries first + +include: + - postgres.client + +{%- endif %} + +# Ensure that Salt is able to use postgres modules + +postgres-reload-modules: + test.nop: + - reload_modules: True + +# User states + +{%- for name, user in postgres.users|dictsort() %} + +{{ format_state(name, 'postgres_user', user) }} + - require: + - test: postgres-reload-modules + +{%- endfor %} + +# Tablespace states + +{%- for name, tblspace in postgres.tablespaces|dictsort() %} + +{{ format_state(name, 'postgres_tablespace', tblspace) }} + - require: + - test: postgres-reload-modules + {%- if 'owner' in tblspace %} + - postgres_user: postgres_user-{{ tblspace.owner }} + {%- endif %} + +{%- endfor %} + +# Database states + +{%- for name, db in postgres.databases|dictsort() %} + +{{ format_state(name, 'postgres_database', db) }} + - require: + - test: postgres-reload-modules + {%- if 'owner' in db %} + - postgres_user: postgres_user-{{ db.owner }} + {%- endif %} + {%- if 'tablespace' in db %} + - postgres_tablespace: postgres_tablespace-{{ db.tablespace }} + {%- endif %} + +{%- endfor %} + +# Schema states + +{%- for name, schema in postgres.schemas|dictsort() %} + +{{ format_state(name, 'postgres_schema', schema) }} + - require: + - test: postgres-reload-modules + {%- if 'owner' in schema %} + - postgres_user: postgres_user-{{ schema.owner }} + {%- endif %} + +{%- endfor %} + +# Extension states + +{%- for name, extension in postgres.extensions|dictsort() %} + +{{ format_state(name, 'postgres_extension', extension) }} + - require: + - test: postgres-reload-modules + {%- if 'maintenance_db' in extension %} + - postgres_database: postgres_database-{{ extension.maintenance_db }} + {%- endif %} + {%- if 'schema' in extension %} + - postgres_schema: postgres_schema-{{ extension.schema }} + {%- endif %} + +{%- endfor %} From e474debefe41ab8515b96b019b9a5bf76cfdd0e4 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Thu, 3 Nov 2016 18:30:41 +0200 Subject: [PATCH 06/42] Remove outdated modules for managing PostgreSQL tablespaces --- _modules/postgres_ext.py | 257 --------------------------------- _states/postgres_tablespace.py | 144 ------------------ 2 files changed, 401 deletions(-) delete mode 100644 _modules/postgres_ext.py delete mode 100644 _states/postgres_tablespace.py diff --git a/_modules/postgres_ext.py b/_modules/postgres_ext.py deleted file mode 100644 index 8712028..0000000 --- a/_modules/postgres_ext.py +++ /dev/null @@ -1,257 +0,0 @@ -from __future__ import absolute_import -import logging -try: - import pipes - import csv - HAS_ALL_IMPORTS = True -except ImportError: - HAS_ALL_IMPORTS = False - -# All this can be removed when we merge this stuff upstream -import salt.utils - -log = logging.getLogger(__name__) - -def __virtual__(): - ''' - Only load this module if the postgres module is already loaded - ''' - if all((salt.utils.which('psql'), HAS_ALL_IMPORTS)): - return True - return False - -# Copied directly from salt/modules/postgres.py, remove when upstreaming -def _run_psql(cmd, runas=None, password=None, host=None, port=None, user=None): - ''' - Helper function to call psql, because the password requirement - makes this too much code to be repeated in each function below - ''' - kwargs = { - 'reset_system_locale': False, - 'clean_env': True, - } - if runas is None: - if not host: - host = __salt__['config.option']('postgres.host') - - if user is None: - user = runas - - if runas: - kwargs['runas'] = runas - - if password is None: - password = __salt__['config.option']('postgres.pass') - if password is not None: - pgpassfile = salt.utils.mkstemp(text=True) - with salt.utils.fopen(pgpassfile, 'w') as fp_: - fp_.write('{0}:{1}:*:{2}:{3}'.format( - 'localhost' if not host or host.startswith('/') else host, - port if port else '*', - user if user else '*', - password, - )) - __salt__['file.chown'](pgpassfile, runas, '') - kwargs['env'] = {'PGPASSFILE': pgpassfile} - - ret = __salt__['cmd.run_all'](cmd, python_shell=False, **kwargs) - - if ret.get('retcode', 0) != 0: - log.error('Error connecting to Postgresql server') - if password is not None and not __salt__['file.remove'](pgpassfile): - log.warning('Remove PGPASSFILE failed') - - return ret - - -def _connection_defaults(user=None, host=None, port=None, maintenance_db=None, - password=None): - ''' - Returns a tuple of (user, host, port, db) with config, pillar, or default - values assigned to missing values. - ''' - if not user: - user = __salt__['config.option']('postgres.user') - if not host: - host = __salt__['config.option']('postgres.host') - if not port: - port = __salt__['config.option']('postgres.port') - if not maintenance_db: - maintenance_db = __salt__['config.option']('postgres.maintenance_db') - if password is None: - password = __salt__['config.option']('postgres.pass') - - return (user, host, port, maintenance_db, password) - - -def _psql_cmd(*args, **kwargs): - ''' - Return string with fully composed psql command. - Accept optional keyword arguments: user, host and port as well as any - number or positional arguments to be added to the end of command. - ''' - (user, host, port, maintenance_db, password) = _connection_defaults( - kwargs.get('user'), - kwargs.get('host'), - kwargs.get('port'), - kwargs.get('maintenance_db'), - kwargs.get('password')) - - cmd = [salt.utils.which('psql'), - '--no-align', - '--no-readline', - '--no-password'] # It is never acceptable to issue a password prompt. - if user: - cmd += ['--username', user] - if host: - cmd += ['--host', host] - if port: - cmd += ['--port', str(port)] - if not maintenance_db: - maintenance_db = 'postgres' - cmd += ['--dbname', maintenance_db] - cmd += args - cmdstr = ' '.join([pipes.quote(c) for c in cmd]) - return cmdstr - - -def _psql_prepare_and_run(cmd, - host=None, - port=None, - maintenance_db=None, - password=None, - runas=None, - user=None): - rcmd = _psql_cmd( - host=host, user=user, port=port, - maintenance_db=maintenance_db, password=password, - *cmd) - cmdret = _run_psql( - rcmd, runas=runas, password=password, host=host, port=port, user=user) - return cmdret - -def tablespace_list(user=None, host=None, port=None, maintenance_db=None, - password=None, runas=None): - ''' - Return dictionary with information about tablespaces of a Postgres server. - CLI Example: - .. code-block:: bash - salt '*' postgres_ext.tablespace_list - ''' - - ret = {} - - query = ( - 'SELECT spcname as "Name", pga.rolname as "Owner", spcacl as "ACL", ' - 'spcoptions as "Opts", pg_tablespace_location(pgts.oid) as "Location" ' - 'FROM pg_tablespace pgts, pg_roles pga WHERE pga.oid = pgts.spcowner' - ) - - rows = __salt__['postgres.psql_query'](query, runas=runas, host=host, - user=user, port=port, - maintenance_db=maintenance_db, - password=password) - - for row in rows: - ret[row['Name']] = row - ret[row['Name']].pop('Name') - - return ret - - -def tablespace_exists(name, user=None, host=None, port=None, maintenance_db=None, - password=None, runas=None): - ''' - Checks if a tablespace exists on the Postgres server. - CLI Example: - .. code-block:: bash - salt '*' postgres_ext.tablespace_exists 'dbname' - ''' - - tablespaces = tablespace_list(user=user, host=host, port=port, - maintenance_db=maintenance_db, - password=password, runas=runas) - return name in tablespaces - - -def tablespace_create(name, location, owner=None, user=None, host=None, port=None, - maintenance_db=None, password=None, runas=None): - ''' - Adds a tablespace to the Postgres server. - - CLI Example: - - .. code-block:: bash - - salt '*' postgres_ext.tablespace_create tablespacename '/path/datadir' - ''' - query = 'CREATE TABLESPACE "{0}" LOCATION \'{1}\''.format(name, location) - if owner is not None: - query += ' OWNER "{1}"'.format(owner) - - # Execute the command - ret = _psql_prepare_and_run(['-c', query], - user=user, host=host, port=port, - maintenance_db=maintenance_db, - password=password, runas=runas) - return ret['retcode'] == 0 - - -def tablespace_alter(name, user=None, host=None, port=None, maintenance_db=None, - password=None, new_name=None, new_owner=None, - set_option=None, reset_option=None, runas=None): - ''' - Change tablespace name, owner, or options. - CLI Example: - .. code-block:: bash - salt '*' postgres_ext.tablespace_alter tsname new_owner=otheruser - salt '*' postgres_ext.tablespace_alter index_space new_name=fast_raid - salt '*' postgres_ext.tablespace_alter tsname reset_option=seq_page_cost - ''' - if not any([new_name, new_owner, set_option, reset_option]): - return True # Nothing todo? - - queries = [] - - if new_name: - queries.append('ALTER TABLESPACE "{}" RENAME TO "{}"'.format( - name, new_name)) - if new_owner: - queries.append('ALTER TABLESPACE "{}" OWNER TO "{}"'.format( - name, new_owner)) - if set_option: - queries.append('ALTER TABLESPACE "{}" SET ({} = {})'.format( - name, set_option[0], set_option[1])) - if reset_option: - queries.append('ALTER TABLESPACE "{}" RESET ({})'.format( - name, reset_option)) - - for query in queries: - ret = _psql_prepare_and_run(['-c', query], - user=user, host=host, port=port, - maintenance_db=maintenance_db, - password=password, runas=runas) - if ret['retcode'] != 0: - return False - - return True - - -def tablespace_remove(name, user=None, host=None, port=None, - maintenance_db=None, password=None, runas=None): - ''' - Removes a tablespace from the Postgres server. - CLI Example: - .. code-block:: bash - salt '*' postgres_ext.tablespace_remove tsname - ''' - query = 'DROP TABLESPACE "{}"'.format(name) - ret = _psql_prepare_and_run(['-c', query], - user=user, - host=host, - port=port, - runas=runas, - maintenance_db=maintenance_db, - password=password) - return ret['retcode'] == 0 - diff --git a/_states/postgres_tablespace.py b/_states/postgres_tablespace.py deleted file mode 100644 index bf0953c..0000000 --- a/_states/postgres_tablespace.py +++ /dev/null @@ -1,144 +0,0 @@ -''' -Management of PostgreSQL tablespace -================================== -The postgres_tablespace module is used to create and manage Postgres -tablespaces. -Tablespaces can be set as either absent or present. -.. code-block:: yaml - ssd-tablespace: - postgres_tablespace.present: - - name: indexes - - path: -''' -from __future__ import absolute_import - - -def __virtual__(): - ''' - Only load if the postgres_ext module is present - ''' - return 'postgres_ext.tablespace_exists' in __salt__ - -def present(name, - directory, - options=None, - owner=None, - user=None, - maintenance_db=None, - db_password=None, - db_host=None, - db_port=None, - db_user=None): - ''' - Ensure that the named tablespace is present with the specified properties. - For more information about all of these options see man create_tablespace(1) - name - The name of the tablespace to manage - directory - The directory where the tablespace will be located - db_user - database username if different from config or defaul - db_password - user password if any password for a specified user - db_host - Database host if different from config or default - db_port - Database port if different from config or default - user - System user all operations should be performed on behalf of - .. versionadded:: Beryllium - ''' - ret = {'name': name, - 'changes': {}, - 'result': True, - 'comment': 'Tablespace {0} is already present'.format(name)} - dbargs = { - 'maintenance_db': maintenance_db, - 'runas': user, - 'host': db_host, - 'user': db_user, - 'port': db_port, - 'password': db_password, - } - tblspaces = __salt__['postgres_ext.tablespace_list'](**dbargs) - if name not in tblspaces: - # not there, create it - if __opts__['test']: - ret['result'] = None - ret['comment'] = 'Tablespace {0} is set to be created'.format(name) - return ret - if __salt__['postgres_ext.tablespace_create'](name, directory, **dbargs): - ret['comment'] = 'The tablespace {0} has been created'.format(name) - ret['changes'][name] = 'Present' - return ret - - # already exists, make sure it's got the right path - if tblspaces[name]['Location'] != directory: - ret['comment'] = 'Tablespace {0} isn\'t at the right path'.format( - name) - ret['result'] = False - return ret # This isn't changeable, they need to remove/remake - - if (owner and not tblspaces[name]['Owner'] == owner): - if __opts__['test']: - ret['result'] = None - ret['comment'] = 'Tablespace {0} owner to be altered'.format(name) - return ret - if __salt__['postgres_ext.tablespace_alter'](name, new_owner=owner): - ret['comment'] = 'Tablespace {0} owner changed'.format(name) - ret['result'] = True - - return ret - - -def absent(name, - user=None, - maintenance_db=None, - db_password=None, - db_host=None, - db_port=None, - db_user=None): - ''' - Ensure that the named database is absent - name - The name of the database to remove - db_user - database username if different from config or defaul - db_password - user password if any password for a specified user - db_host - Database host if different from config or default - db_port - Database port if different from config or default - user - System user all operations should be performed on behalf of - .. versionadded:: Beryllium - ''' - ret = {'name': name, - 'changes': {}, - 'result': True, - 'comment': ''} - - db_args = { - 'maintenance_db': maintenance_db, - 'runas': user, - 'host': db_host, - 'user': db_user, - 'port': db_port, - 'password': db_password, - } - #check if tablespace exists and remove it - if __salt__['postgres_ext.tablespace_exists'](name, **db_args): - if __opts__['test']: - ret['result'] = None - ret['comment'] = 'Tablespace {0} is set to be removed'.format(name) - return ret - if __salt__['postgres_ext.tablespace_remove'](name, **db_args): - ret['comment'] = 'Tablespace {0} has been removed'.format(name) - ret['changes'][name] = 'Absent' - return ret - - # fallback - ret['comment'] = 'Tablespace {0} is not present, so it cannot ' \ - 'be removed'.format(name) - return ret From ab38a069fa047c01479c9b47a0edbb0391126be3 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Thu, 1 Dec 2016 16:11:37 +0200 Subject: [PATCH 07/42] Workaround a bug in saltstack/salt#37935 for release v2016.11.0 --- postgres/manage.sls | 4 ++++ postgres/osmap.yaml | 9 +++++++++ postgres/server.sls | 21 +++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/postgres/manage.sls b/postgres/manage.sls index ea31836..66f5865 100644 --- a/postgres/manage.sls +++ b/postgres/manage.sls @@ -8,6 +8,10 @@ include: - postgres.client + {%- if 'server_bins' in postgres and grains['saltversion'] == '2016.11.0' %} + # FIXME: Salt v2016.11.0 bug https://github.com/saltstack/salt/issues/37935 + - postgres.server + {%- endif %} {%- endif %} diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index ebef733..128bfe3 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -69,6 +69,15 @@ RedHat: - psql - reindexdb - vacuumdb + server_bins: + - initdb + - pg_controldata + - pg_ctl + - pg_resetxlog + - postgres + - postgresql{{ release }}-check-db-dir + - postgresql{{ release }}-setup + - postmaster {% else %} diff --git a/postgres/server.sls b/postgres/server.sls index 5efb04f..a553dae 100644 --- a/postgres/server.sls +++ b/postgres/server.sls @@ -92,3 +92,24 @@ postgresql-tablespace-dir-{{ name }}: - group {%- endfor %} + +{%- if 'bin_dir' in postgres %} + +# Make server binaries available in $PATH + + {%- for bin in postgres.server_bins %} + + {%- set path = salt['file.join'](postgres.bin_dir, bin) %} + +{{ bin }}: + alternatives.install: + - link: {{ salt['file.join']('/usr/bin', bin) }} + - path: {{ path }} + - priority: 30 + - onlyif: test -f {{ path }} + - require: + - pkg: postgresql-server + + {%- endfor %} + +{%- endif %} From a291d40ef15483132ce7ac04bfa06e253861a770 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Mon, 19 Dec 2016 12:29:14 +0200 Subject: [PATCH 08/42] Fix and set useful backup extension for postgresql.conf file --- pillar.example | 21 +++++++++++++-------- postgres/defaults.yaml | 38 +++++++++++++++++++++----------------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/pillar.example b/pillar.example index 13e9887..98a4d10 100644 --- a/pillar.example +++ b/pillar.example @@ -14,6 +14,16 @@ postgres: - postgresql-contrib - postgresql-plpython + # This section will append your configuration to postgresql.conf. + postgresconf: | + listen_addresses = 'localhost,*' + + # Backup extension for postgresql.conf file, defaults to `.bak`. + # Set to False to stop creation of backup on postgresql.conf changes. + {%- if 'status.time' in salt.keys() %} + postgresconf_backup: ".backup@{{ salt['status.time']('%y-%m-%d_%H:%M:%S') }}" + {%- endif %} + # Path to the `pg_hba.conf` file Jinja template on Salt Fileserver pg_hba.conf: salt://postgres/templates/pg_hba.conf.j2 @@ -87,11 +97,13 @@ postgres: absentUser: ensure: absent + # tablespaces to be created tablespaces: my_space: directory: /srv/my_tablespace owner: localUser + # databases to be created databases: db1: owner: 'localUser' @@ -118,11 +130,4 @@ postgres: maintenance_db: db1 #postgis: {} - # Backup extension defaults to .bak if postgresconf_backup is True. - # Set to False to stop creation of backup on postgresql.conf changes. - postgresconf_backup: True - # This section will append your configuration to postgresql.conf. - postgresconf: | - listen_addresses = 'localhost,*' - -# vim: ft=yaml:sw=2 +# vim: ft=yaml ts=2 sts=2 sw=2 et diff --git a/postgres/defaults.yaml b/postgres/defaults.yaml index 9172a9b..a3bc212 100644 --- a/postgres/defaults.yaml +++ b/postgres/defaults.yaml @@ -11,23 +11,6 @@ postgres: python: python-psycopg2 user: postgres group: postgres - conf_dir: /var/lib/pgsql/data - pg_hba.conf: salt://postgres/templates/pg_hba.conf.j2 - acls: - # "local" is for Unix domain socket connections only - - ['local', 'all', 'all', 'peer'] - # IPv4 local connections: - - ['host', 'all', 'all', '127.0.0.1/32', 'md5'] - # IPv6 local connections: - - ['host', 'all', 'all', '::1/128', 'md5'] - service: postgresql - users: {} - tablespaces: {} - databases: {} - schemas: {} - extensions: {} - postgresconf_backup: True - postgresconf: "" # if prepare_cluster is over-ridden in any of: # - osmap.yaml @@ -39,3 +22,24 @@ postgres: command: service postgresql initdb test: test -f /var/lib/pgsql/data/PG_VERSION env: {} + + conf_dir: /var/lib/pgsql/data + postgresconf: "" + postgresconf_backup: '.bak' + + pg_hba.conf: salt://postgres/templates/pg_hba.conf.j2 + acls: + # "local" is for Unix domain socket connections only + - ['local', 'all', 'all', 'peer'] + # IPv4 local connections: + - ['host', 'all', 'all', '127.0.0.1/32', 'md5'] + # IPv6 local connections: + - ['host', 'all', 'all', '::1/128', 'md5'] + + service: postgresql + + users: {} + tablespaces: {} + databases: {} + schemas: {} + extensions: {} From 3a022aa5b7ac2e14c2026d22747bf2b34ac3dd23 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Tue, 20 Dec 2016 12:05:32 +0200 Subject: [PATCH 09/42] Correct `listen_addresses` option example for postgresql.conf file --- pillar.example | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pillar.example b/pillar.example index 98a4d10..ddc2be7 100644 --- a/pillar.example +++ b/pillar.example @@ -14,9 +14,10 @@ postgres: - postgresql-contrib - postgresql-plpython - # This section will append your configuration to postgresql.conf. + # Append the lines under this item to your postgresql.conf file. + # Pay attention to indent exactly with 4 spaces for all lines. postgresconf: | - listen_addresses = 'localhost,*' + listen_addresses = '*' # listen on all interfaces # Backup extension for postgresql.conf file, defaults to `.bak`. # Set to False to stop creation of backup on postgresql.conf changes. From d4ed4be16ecb181e73a5fdc25de109aad1316698 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Tue, 3 Jan 2017 11:54:00 +0200 Subject: [PATCH 10/42] Attempt to start and manage PostgreSQL service even if init system fails --- postgres/codenamemap.yaml | 1 - postgres/defaults.yaml | 9 +---- postgres/map.jinja | 7 +--- postgres/osmajorreleasemap.yaml | 67 --------------------------------- postgres/osmap.yaml | 8 +++- postgres/server.sls | 62 +++++++++++++++++++++--------- 6 files changed, 53 insertions(+), 101 deletions(-) delete mode 100644 postgres/osmajorreleasemap.yaml diff --git a/postgres/codenamemap.yaml b/postgres/codenamemap.yaml index eaf18f3..eb98aa1 100644 --- a/postgres/codenamemap.yaml +++ b/postgres/codenamemap.yaml @@ -31,7 +31,6 @@ command: pg_createcluster {{ version }} main test: test -f /var/lib/postgresql/{{ version }}/main/PG_VERSION && test -f /etc/postgresql/{{ version }}/main/postgresql.conf user: root - env: {} {% endmacro %} diff --git a/postgres/defaults.yaml b/postgres/defaults.yaml index a3bc212..67f643f 100644 --- a/postgres/defaults.yaml +++ b/postgres/defaults.yaml @@ -12,15 +12,10 @@ postgres: user: postgres group: postgres - # if prepare_cluster is over-ridden in any of: - # - osmap.yaml - # - oscodenamemap.yaml - # - osfingermap.yaml - # you will have to specify a complete dictionary. prepare_cluster: - user: root - command: service postgresql initdb + command: initdb --pgdata=/var/lib/pgsql/data test: test -f /var/lib/pgsql/data/PG_VERSION + user: postgres env: {} conf_dir: /var/lib/pgsql/data diff --git a/postgres/map.jinja b/postgres/map.jinja index 75d02c8..1b947ac 100644 --- a/postgres/map.jinja +++ b/postgres/map.jinja @@ -1,7 +1,6 @@ {% import_yaml "postgres/defaults.yaml" as defaults %} {% import_yaml "postgres/osmap.yaml" as osmap %} {% import_yaml "postgres/codenamemap.yaml" as oscodenamemap %} -{% import_yaml "postgres/osmajorreleasemap.yaml" as osmajorreleasemap %} {% set postgres = salt['grains.filter_by']( defaults, @@ -11,11 +10,7 @@ merge=salt['grains.filter_by']( oscodenamemap, grain='oscodename', - merge=salt['grains.filter_by']( - osmajorreleasemap, - grain='osmajorrelease', - merge=salt['pillar.get']('postgres', {}), - ), + merge=salt['pillar.get']('postgres', {}), ), ), base='postgres', diff --git a/postgres/osmajorreleasemap.yaml b/postgres/osmajorreleasemap.yaml deleted file mode 100644 index b49590d..0000000 --- a/postgres/osmajorreleasemap.yaml +++ /dev/null @@ -1,67 +0,0 @@ -{% import_yaml "postgres/repo.yaml" as repo %} - -{% if grains['os_family'] == 'RedHat' %} - -### RedHat releases - - {% if repo.use_upstream_repo %} - - {% set data_dir = '/var/lib/pgsql/' ~ repo.version ~ '/data' %} - -# PostgreSQL from upstream repository - -default: - prepare_cluster: - user: postgres - command: /usr/pgsql-{{ repo.version }}/bin/initdb -D {{ data_dir }} - test: test -f {{ data_dir }}/PG_VERSION - env: {} -'6': - prepare_cluster: - user: root - command: service postgresql-{{ repo.version }} initdb - test: test -f {{ data_dir }}/PG_VERSION - env: {} -'7': - prepare_cluster: - user: root - command: /usr/pgsql-{{ repo.version }}/bin/postgresql95-setup initdb - test: test -f {{ data_dir }}/PG_VERSION - env: {} - - {% else %} - - {% set data_dir = '/var/lib/pgsql/data' %} - -# PostgreSQL from OS repositories - -default: - prepare_cluster: - user: postgres - command: initdb -D {{ data_dir }} - test: test -f {{ data_dir }}/PG_VERSION - env: {} -'6': - prepare_cluster: - user: root - command: service postgresql initdb - test: test -f {{ data_dir }}/PG_VERSION - env: {} -'7': - prepare_cluster: - user: root - command: postgresql-setup initdb - test: test -f {{ data_dir }}/PG_VERSION - env: {} - - {% endif %} - -{% else %} - -### Empty YAML, no settings for other OS - -{} - -{% endif %} - -# vim: ft=sls diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 128bfe3..f41a93e 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -5,10 +5,8 @@ Arch: conf_dir: /var/lib/postgres/data prepare_cluster: - user: postgres command: initdb -D /var/lib/postgresql/data test: test -f /var/lib/postgres/data/PG_VERSION - env: {} pkg_client: postgresql pkg_dev: postgresql @@ -37,11 +35,17 @@ RedHat: {% if repo.use_upstream_repo %} + {% set data_dir = '/var/lib/pgsql/' ~ repo.version ~ '/data' %} + pkg: postgresql{{ release }}-server pkg_client: postgresql{{ release }} conf_dir: /var/lib/pgsql/{{ repo.version }}/data service: postgresql-{{ repo.version }} + prepare_cluster: + command: initdb --pgdata='{{ data_dir }}' + test: test -f '{{ data_dir }}/PG_VERSION' + # Directory containing PostgreSQL client executables bin_dir: /usr/pgsql-{{ repo.version }}/bin client_bins: diff --git a/postgres/server.sls b/postgres/server.sls index a553dae..e92ea3f 100644 --- a/postgres/server.sls +++ b/postgres/server.sls @@ -20,6 +20,27 @@ postgresql-server: - pkgrepo: postgresql-repo {%- endif %} +{%- if 'bin_dir' in postgres %} + +# Make server binaries available in $PATH + + {%- for bin in postgres.server_bins %} + + {%- set path = salt['file.join'](postgres.bin_dir, bin) %} + +{{ bin }}: + alternatives.install: + - link: {{ salt['file.join']('/usr/bin', bin) }} + - path: {{ path }} + - priority: 30 + - onlyif: test -f {{ path }} + - require: + - pkg: postgresql-server + + {%- endfor %} + +{%- endif %} + postgresql-cluster-prepared: cmd.run: - name: {{ postgres.prepare_cluster.command }} @@ -93,23 +114,28 @@ postgresql-tablespace-dir-{{ name }}: {%- endfor %} -{%- if 'bin_dir' in postgres %} - -# Make server binaries available in $PATH - - {%- for bin in postgres.server_bins %} - - {%- set path = salt['file.join'](postgres.bin_dir, bin) %} - -{{ bin }}: - alternatives.install: - - link: {{ salt['file.join']('/usr/bin', bin) }} - - path: {{ path }} - - priority: 30 - - onlyif: test -f {{ path }} - - require: - - pkg: postgresql-server - - {%- endfor %} +# An attempt to launch PostgreSQL with `pg_ctl` if service failed to start +# with init system or Salt unable to load the `service` state module +postgresql-start: + cmd.run: + - name: pg_ctl -D {{ postgres.conf_dir }} -l logfile start + - runas: {{ postgres.user }} + - unless: + - ps -p $(head -n 1 {{ postgres.conf_dir }}/postmaster.pid) 2>/dev/null + - onfail: + - service: postgresql-running +# Try to enable PostgreSQL in "manual" way for systemd and RedHat-based distros. +# The packages for other OS (i.e. `*.deb`) should do it automatically by default +postgresql-enable: + cmd.run: +{%- if salt['file.file_exists']('/bin/systemctl') %} + - name: systemctl enable {{ postgres.service }} +{%- elif salt['cmd.which']('chkconfig') %} + - name: chkconfig {{ postgres.service }} on +{%- else %} + # Nothing to do + - name: 'true' {%- endif %} + - onchanges: + - cmd: postgresql-start From 87f6cd880ca637984268e3bbe4ef7078f47ced08 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Fri, 6 Jan 2017 16:37:07 +0200 Subject: [PATCH 11/42] Make clean state run by explicitly checking if init system is available --- postgres/server.sls | 40 ++++++++++++++++++------------- postgres/templates/pg_hba.conf.j2 | 4 +--- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/postgres/server.sls b/postgres/server.sls index e92ea3f..3c1c170 100644 --- a/postgres/server.sls +++ b/postgres/server.sls @@ -88,17 +88,11 @@ postgresql-pg_hba: - user: {{ postgres.user }} - group: {{ postgres.group }} - mode: 600 + - defaults: + acls: {{ postgres.acls }} - require: - file: postgresql-config-dir -postgresql-running: - service.running: - - name: {{ postgres.service }} - - enable: True - - reload: True - - watch: - - file: postgresql-pg_hba - {%- for name, tblspace in postgres.tablespaces|dictsort() %} postgresql-tablespace-dir-{{ name }}: @@ -114,28 +108,42 @@ postgresql-tablespace-dir-{{ name }}: {%- endfor %} -# An attempt to launch PostgreSQL with `pg_ctl` if service failed to start -# with init system or Salt unable to load the `service` state module +{%- if grains['init'] != 'unknown' %} + +postgresql-running: + service.running: + - name: {{ postgres.service }} + - enable: True + - reload: True + - watch: + - file: postgresql-pg_hba + +{%- else %} + +# An attempt to launch PostgreSQL with `pg_ctl` if Salt was unable to +# detect local init system (`service` module would fail in this case) + postgresql-start: cmd.run: - name: pg_ctl -D {{ postgres.conf_dir }} -l logfile start - runas: {{ postgres.user }} - unless: - ps -p $(head -n 1 {{ postgres.conf_dir }}/postmaster.pid) 2>/dev/null - - onfail: - - service: postgresql-running # Try to enable PostgreSQL in "manual" way for systemd and RedHat-based distros. # The packages for other OS (i.e. `*.deb`) should do it automatically by default + postgresql-enable: cmd.run: -{%- if salt['file.file_exists']('/bin/systemctl') %} + {%- if salt['file.file_exists']('/bin/systemctl') %} - name: systemctl enable {{ postgres.service }} -{%- elif salt['cmd.which']('chkconfig') %} + {%- elif salt['cmd.which']('chkconfig') %} - name: chkconfig {{ postgres.service }} on -{%- else %} + {%- else %} # Nothing to do - name: 'true' -{%- endif %} + {%- endif %} - onchanges: - cmd: postgresql-start + +{%- endif %} diff --git a/postgres/templates/pg_hba.conf.j2 b/postgres/templates/pg_hba.conf.j2 index 560c45e..11f4cad 100644 --- a/postgres/templates/pg_hba.conf.j2 +++ b/postgres/templates/pg_hba.conf.j2 @@ -1,5 +1,3 @@ -{%- from "postgres/map.jinja" import postgres with context -%} - ###################################################################### # ATTENTION! Managed by SaltStack. # # DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN! # @@ -22,7 +20,7 @@ local all postgres peer # TYPE DATABASE USER ADDRESS METHOD -{%- for acl in postgres.acls %} +{%- for acl in acls %} {%- if acl|first() == 'local' %} {%- if acl|length() == 3 %} From d48f6f2a50535acaf01a343d6a5723be26268905 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Fri, 6 Jan 2017 16:54:33 +0200 Subject: [PATCH 12/42] Make sure the service is enabled on "deb"-based operating systems --- postgres/server.sls | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/postgres/server.sls b/postgres/server.sls index 3c1c170..b9930db 100644 --- a/postgres/server.sls +++ b/postgres/server.sls @@ -130,8 +130,9 @@ postgresql-start: - unless: - ps -p $(head -n 1 {{ postgres.conf_dir }}/postmaster.pid) 2>/dev/null -# Try to enable PostgreSQL in "manual" way for systemd and RedHat-based distros. -# The packages for other OS (i.e. `*.deb`) should do it automatically by default +# Try to enable PostgreSQL in "manual" way if Salt `service` state module +# is currently not available (e.g. during Docker or Packer build when is no init +# system running) postgresql-enable: cmd.run: @@ -139,6 +140,8 @@ postgresql-enable: - name: systemctl enable {{ postgres.service }} {%- elif salt['cmd.which']('chkconfig') %} - name: chkconfig {{ postgres.service }} on + {%- elif salt['file.file_exists']('/usr/sbin/update-rc.d') %} + - name: update-rc.d {{ service }} defaults {%- else %} # Nothing to do - name: 'true' From 186e972075619bb4b57e3f392dbfdc0192d65472 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Sat, 7 Jan 2017 15:11:26 +0200 Subject: [PATCH 13/42] Use ``bake_image`` Pillar setting to control PG startup behavior --- pillar.example | 17 ++++++++++++++--- postgres/defaults.yaml | 2 ++ postgres/server.sls | 15 ++++++++------- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/pillar.example b/pillar.example index ddc2be7..e877314 100644 --- a/pillar.example +++ b/pillar.example @@ -19,7 +19,7 @@ postgres: postgresconf: | listen_addresses = '*' # listen on all interfaces - # Backup extension for postgresql.conf file, defaults to `.bak`. + # Backup extension for postgresql.conf file, defaults to ``.bak``. # Set to False to stop creation of backup on postgresql.conf changes. {%- if 'status.time' in salt.keys() %} postgresconf_backup: ".backup@{{ salt['status.time']('%y-%m-%d_%H:%M:%S') }}" @@ -28,7 +28,7 @@ postgres: # Path to the `pg_hba.conf` file Jinja template on Salt Fileserver pg_hba.conf: salt://postgres/templates/pg_hba.conf.j2 - # This section covers ACL management in the `pg_hba.conf` file. + # This section covers ACL management in the ``pg_hba.conf`` file. # acls list controls: which hosts are allowed to connect, how clients # are authenticated, which PostgreSQL user names they can use, which # databases they can access. Records take one of these forms: @@ -48,9 +48,20 @@ postgres: # PostgreSQL service name service: postgresql + {%- if grains['init'] == 'unknown' %} + + # If Salt is unable to detect init system running in the scope of state run, + # probably we are trying to bake a container/VM image with PostgreSQL. + # Use ``service`` setting to contol how PostgreSQL will be started: if set to + # ``True`` the raw ``pg_ctl`` will be utilized instead of packaged init + # script, job or unit run with Salt ``service`` state. + bake_image: True + + {%- endif %} + # Create/remove users, tablespaces, databases, schema and extensions. # Each of these dictionaries contains PostgreSQL entities which - # mapped to the `postgres_*` Salt states with arguments. See the Salt + # mapped to the ``postgres_*`` Salt states with arguments. See the Salt # documentaion to get all supported argument for a particular state. # # Format is the following: diff --git a/postgres/defaults.yaml b/postgres/defaults.yaml index 67f643f..d421aff 100644 --- a/postgres/defaults.yaml +++ b/postgres/defaults.yaml @@ -33,6 +33,8 @@ postgres: service: postgresql + bake_image: False + users: {} tablespaces: {} databases: {} diff --git a/postgres/server.sls b/postgres/server.sls index b9930db..8c12ba4 100644 --- a/postgres/server.sls +++ b/postgres/server.sls @@ -108,7 +108,9 @@ postgresql-tablespace-dir-{{ name }}: {%- endfor %} -{%- if grains['init'] != 'unknown' %} +{%- if not postgres.bake_image %} + +# Start PostgreSQL server using OS init postgresql-running: service.running: @@ -120,8 +122,7 @@ postgresql-running: {%- else %} -# An attempt to launch PostgreSQL with `pg_ctl` if Salt was unable to -# detect local init system (`service` module would fail in this case) +# An attempt to launch PostgreSQL with `pg_ctl` during an image preparation postgresql-start: cmd.run: @@ -129,10 +130,10 @@ postgresql-start: - runas: {{ postgres.user }} - unless: - ps -p $(head -n 1 {{ postgres.conf_dir }}/postmaster.pid) 2>/dev/null + - require: + - file: postgresql-pg_hba -# Try to enable PostgreSQL in "manual" way if Salt `service` state module -# is currently not available (e.g. during Docker or Packer build when is no init -# system running) +# Try to enable PostgreSQL in "manual" way when baking an image postgresql-enable: cmd.run: @@ -146,7 +147,7 @@ postgresql-enable: # Nothing to do - name: 'true' {%- endif %} - - onchanges: + - require: - cmd: postgresql-start {%- endif %} From 783041b8cc823b46e1949f517549830ab772e1f6 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Sun, 8 Jan 2017 17:41:12 +0200 Subject: [PATCH 14/42] Separate states for baking VM or container image --- postgres/server/image.sls | 45 ++++++++++++++++++++++ postgres/{server.sls => server/init.sls} | 48 +++++++----------------- 2 files changed, 59 insertions(+), 34 deletions(-) create mode 100644 postgres/server/image.sls rename postgres/{server.sls => server/init.sls} (72%) diff --git a/postgres/server/image.sls b/postgres/server/image.sls new file mode 100644 index 0000000..1cd3d5b --- /dev/null +++ b/postgres/server/image.sls @@ -0,0 +1,45 @@ +{%- from "postgres/map.jinja" import postgres with context -%} + +# This state is used to launch PostgreSQL and enable it on "boot" with `pg_ctl` +# during an image (Docker, Virtual Appliance, AMI) preparation + +{%- if postgres.bake_image %} + +include: + - postgres.server + +# An attempt to start PostgreSQL with `pg_ctl` + +postgresql-start: + cmd.run: + - name: pg_ctl -D {{ postgres.conf_dir }} -l logfile start + - runas: {{ postgres.user }} + - unless: + - ps -p $(head -n 1 {{ postgres.conf_dir }}/postmaster.pid) 2>/dev/null + - require: + - file: postgresql-pg_hba + +# Try to enable PostgreSQL in "manual" way + +postgresql-enable: + cmd.run: + {%- if salt['file.file_exists']('/bin/systemctl') %} + - name: systemctl enable {{ postgres.service }} + {%- elif salt['cmd.which']('chkconfig') %} + - name: chkconfig {{ postgres.service }} on + {%- elif salt['file.file_exists']('/usr/sbin/update-rc.d') %} + - name: update-rc.d {{ service }} defaults + {%- else %} + # Nothing to do + - name: 'true' + {%- endif %} + - require: + - cmd: postgresql-start + +{%- else %} + +postgresql-start: + test.show_notification: + - text: The 'postgres.bake_image' Pillar is disabled (set to 'False'). + +{%- endif %} diff --git a/postgres/server.sls b/postgres/server/init.sls similarity index 72% rename from postgres/server.sls rename to postgres/server/init.sls index 8c12ba4..181840b 100644 --- a/postgres/server.sls +++ b/postgres/server/init.sls @@ -1,11 +1,19 @@ -{%- from "postgres/map.jinja" import postgres with context -%} - -{%- set pkgs = [postgres.pkg] + postgres.pkgs_extra -%} +{%- from "postgres/map.jinja" import postgres with context %} +{%- set includes = [] %} +{%- if postgres.bake_image %} + {%- do includes.append('postgres.server.image') %} +{%- endif %} {%- if postgres.use_upstream_repo -%} + {%- do includes.append('postgres.upstream') %} +{%- endif %} + +{%- set pkgs = [postgres.pkg] + postgres.pkgs_extra %} + +{%- if includes -%} include: - - postgres.upstream + {{ includes|yaml(false)|indent(2) }} {%- endif %} @@ -36,6 +44,8 @@ postgresql-server: - onlyif: test -f {{ path }} - require: - pkg: postgresql-server + - require_in: + - cmd: postgresql-cluster-prepared {%- endfor %} @@ -120,34 +130,4 @@ postgresql-running: - watch: - file: postgresql-pg_hba -{%- else %} - -# An attempt to launch PostgreSQL with `pg_ctl` during an image preparation - -postgresql-start: - cmd.run: - - name: pg_ctl -D {{ postgres.conf_dir }} -l logfile start - - runas: {{ postgres.user }} - - unless: - - ps -p $(head -n 1 {{ postgres.conf_dir }}/postmaster.pid) 2>/dev/null - - require: - - file: postgresql-pg_hba - -# Try to enable PostgreSQL in "manual" way when baking an image - -postgresql-enable: - cmd.run: - {%- if salt['file.file_exists']('/bin/systemctl') %} - - name: systemctl enable {{ postgres.service }} - {%- elif salt['cmd.which']('chkconfig') %} - - name: chkconfig {{ postgres.service }} on - {%- elif salt['file.file_exists']('/usr/sbin/update-rc.d') %} - - name: update-rc.d {{ service }} defaults - {%- else %} - # Nothing to do - - name: 'true' - {%- endif %} - - require: - - cmd: postgresql-start - {%- endif %} From b7771fec8c862dd6d68c0c3b2574bdbdc3f72ea1 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Mon, 9 Jan 2017 16:01:50 +0200 Subject: [PATCH 15/42] Fix requisite for tablespace directories --- postgres/server/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/postgres/server/init.sls b/postgres/server/init.sls index 181840b..0c23ab3 100644 --- a/postgres/server/init.sls +++ b/postgres/server/init.sls @@ -115,6 +115,8 @@ postgresql-tablespace-dir-{{ name }}: - recurse: - user - group + - require: + - pkg: postgresql-server {%- endfor %} From 52d57d357633ea7abbe83e38f0315e84199b6a0e Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Mon, 9 Jan 2017 16:05:48 +0200 Subject: [PATCH 16/42] Describe ``postgres.server.image`` state in README --- README.rst | 46 +++++++++++++++++++++++++++++++-------- pillar.example | 4 ++-- postgres/server/image.sls | 6 ++--- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index b8d547e..bd90d90 100644 --- a/README.rst +++ b/README.rst @@ -16,8 +16,8 @@ Available states ``postgres`` ------------ -Installs and configures both PostgreSQL server and client with creation of -various DB objects in the cluster. +Installs and configures both PostgreSQL server and client with creation of various DB objects in +the cluster. ``postgres.client`` ------------------- @@ -27,8 +27,8 @@ Installs the PostgreSQL client binaries and libraries. ``postgres.manage`` ------------------- -Creates such DB objects as: users, tablespaces, databases, schemas and -extensions. See ``pillar.example`` file for details. +Creates such DB objects as: users, tablespaces, databases, schemas and extensions. +See ``pillar.example`` file for details. ``postgres.python`` ------------------- @@ -38,7 +38,34 @@ Installs the PostgreSQL adapter for Python. ``postgres.server`` ------------------- -Installs the PostgreSQL server package and prepares the DB cluster. +Installs the PostgreSQL server package, prepares the DB cluster and starts the server using +packaged init script, job or unit. + +``postgres.server.image`` +------------------------- + +Installs the PostgreSQL server package, prepares the DB cluster and starts the server by issuing +raw ``pg_ctl`` command. The ``postgres:bake_image`` Pillar toggles this behaviour. For example: + +.. code:: yaml + + postgres: + bake_image: True + +If set ``True``, then it becomes possible to fully provision PostgreSQL with all supported entities +from ``postgres.manage`` state during the build ("baking") of AMI / VM / Container images (using +Packer, Docker or similar tools), i.e. when OS ``init`` process is not available to start the +service and enable it on "boot" of resulting appliance. + +Also it allows to make Docker images with PostgreSQL using functionality being available since Salt +2016.11.0 release: + +.. code:: console + + salt 'minion.with.docker' dockerng.sls_build my-postgres base=centos/systemd mods=postgres + +If a lookup dictionary or Pillar has ``postgres:bake_image`` set ``False`` (this is default), it is +equivalent of applying ``postgres.server`` state. ``postgres.upstream`` --------------------- @@ -46,14 +73,13 @@ Installs the PostgreSQL server package and prepares the DB cluster. Configures the PostgreSQL Official (upstream) repository on target system if applicable. -The state relies on the ``postgres:use_upstream_repo`` Pillar value which could -be set as following: +The state relies on the ``postgres:use_upstream_repo`` Pillar value which could be set as following: * ``True`` (default): adds the upstream repository to install packages from * ``False``: makes sure that the repository configuration is absent -The ``postgres:version`` Pillar controls which version of the PostgreSQL -packages should be installed from the upstream repository. Defaults to ``9.5``. +The ``postgres:version`` Pillar controls which version of the PostgreSQL packages should be +installed from the upstream repository. Defaults to ``9.5``. Testing ======= @@ -79,3 +105,5 @@ Builds and runs tests from scratch. ----------------- Gives you ssh to the vagrant machine for manual testing. + +.. vim: fenc=utf-8 spell spl=en cc=100 tw=99 fo=want sts=2 sw=2 et diff --git a/pillar.example b/pillar.example index e877314..b95d65c 100644 --- a/pillar.example +++ b/pillar.example @@ -52,8 +52,8 @@ postgres: # If Salt is unable to detect init system running in the scope of state run, # probably we are trying to bake a container/VM image with PostgreSQL. - # Use ``service`` setting to contol how PostgreSQL will be started: if set to - # ``True`` the raw ``pg_ctl`` will be utilized instead of packaged init + # Use ``bake_image`` setting to contol how PostgreSQL will be started: if set + # to ``True`` the raw ``pg_ctl`` will be utilized instead of packaged init # script, job or unit run with Salt ``service`` state. bake_image: True diff --git a/postgres/server/image.sls b/postgres/server/image.sls index 1cd3d5b..95838e7 100644 --- a/postgres/server/image.sls +++ b/postgres/server/image.sls @@ -1,7 +1,7 @@ {%- from "postgres/map.jinja" import postgres with context -%} -# This state is used to launch PostgreSQL and enable it on "boot" with `pg_ctl` -# during an image (Docker, Virtual Appliance, AMI) preparation +# This state is used to launch PostgreSQL with ``pg_ctl`` command and enable it +# on "boot" during an image (Docker, Virtual Appliance, AMI) preparation {%- if postgres.bake_image %} @@ -40,6 +40,6 @@ postgresql-enable: postgresql-start: test.show_notification: - - text: The 'postgres.bake_image' Pillar is disabled (set to 'False'). + - text: The 'postgres:bake_image' Pillar is disabled (set to 'False'). {%- endif %} From 69a59b6f63050714df89863a312c7e0085af067c Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Thu, 12 Jan 2017 11:34:41 +0200 Subject: [PATCH 17/42] Skip configuring ACLs in ``pg_hba.conf`` if not provided --- pillar.example | 7 +++++-- postgres/server/init.sls | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pillar.example b/pillar.example index b95d65c..0d294cf 100644 --- a/pillar.example +++ b/pillar.example @@ -41,6 +41,9 @@ postgres: # # The uppercase items must be replaced by actual values. # METHOD could be omitted, 'md5' will be appended by default. + # + # If ``acls`` item value is empty ('', [], null), then the contents of + # ``pg_hba.conf`` file will not be touched at all. acls: - ['local', 'db1', 'localUser'] - ['host', 'db2', 'remoteUser', '192.168.33.0/24'] @@ -52,7 +55,7 @@ postgres: # If Salt is unable to detect init system running in the scope of state run, # probably we are trying to bake a container/VM image with PostgreSQL. - # Use ``bake_image`` setting to contol how PostgreSQL will be started: if set + # Use ``bake_image`` setting to control how PostgreSQL will be started: if set # to ``True`` the raw ``pg_ctl`` will be utilized instead of packaged init # script, job or unit run with Salt ``service`` state. bake_image: True @@ -62,7 +65,7 @@ postgres: # Create/remove users, tablespaces, databases, schema and extensions. # Each of these dictionaries contains PostgreSQL entities which # mapped to the ``postgres_*`` Salt states with arguments. See the Salt - # documentaion to get all supported argument for a particular state. + # documentation to get all supported argument for a particular state. # # Format is the following: # diff --git a/postgres/server/init.sls b/postgres/server/init.sls index 0c23ab3..91fe11d 100644 --- a/postgres/server/init.sls +++ b/postgres/server/init.sls @@ -93,13 +93,17 @@ postgresql-conf: postgresql-pg_hba: file.managed: - name: {{ postgres.conf_dir }}/pg_hba.conf - - source: {{ postgres['pg_hba.conf'] }} - - template: jinja - user: {{ postgres.user }} - group: {{ postgres.group }} - mode: 600 +{%- if postgres.acls %} + - source: {{ postgres['pg_hba.conf'] }} + - template: jinja - defaults: acls: {{ postgres.acls }} +{%- else %} + - replace: False +{%- endif %} - require: - file: postgresql-config-dir From 79b83fe414e0de43b13945d02735457114757092 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Fri, 13 Jan 2017 12:09:45 +0200 Subject: [PATCH 18/42] Make local backup of `pg_hba.conf` file before modification --- pillar.example | 12 ++++++------ postgres/defaults.yaml | 3 ++- postgres/server/init.sls | 22 ++++++++++++++++++---- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/pillar.example b/pillar.example index 0d294cf..4e02719 100644 --- a/pillar.example +++ b/pillar.example @@ -19,12 +19,6 @@ postgres: postgresconf: | listen_addresses = '*' # listen on all interfaces - # Backup extension for postgresql.conf file, defaults to ``.bak``. - # Set to False to stop creation of backup on postgresql.conf changes. - {%- if 'status.time' in salt.keys() %} - postgresconf_backup: ".backup@{{ salt['status.time']('%y-%m-%d_%H:%M:%S') }}" - {%- endif %} - # Path to the `pg_hba.conf` file Jinja template on Salt Fileserver pg_hba.conf: salt://postgres/templates/pg_hba.conf.j2 @@ -48,6 +42,12 @@ postgres: - ['local', 'db1', 'localUser'] - ['host', 'db2', 'remoteUser', '192.168.33.0/24'] + # Backup extension for configuration files, defaults to ``.bak``. + # Set ``False`` to stop creation of backups when config files change. + {%- if 'status.time' in salt.keys() %} + config_backup: ".backup@{{ salt['status.time']('%y-%m-%d_%H:%M:%S') }}" + {%- endif %} + # PostgreSQL service name service: postgresql diff --git a/postgres/defaults.yaml b/postgres/defaults.yaml index d421aff..50f9acc 100644 --- a/postgres/defaults.yaml +++ b/postgres/defaults.yaml @@ -20,7 +20,6 @@ postgres: conf_dir: /var/lib/pgsql/data postgresconf: "" - postgresconf_backup: '.bak' pg_hba.conf: salt://postgres/templates/pg_hba.conf.j2 acls: @@ -31,6 +30,8 @@ postgres: # IPv6 local connections: - ['host', 'all', 'all', '::1/128', 'md5'] + config_backup: '.bak' + service: postgresql bake_image: False diff --git a/postgres/server/init.sls b/postgres/server/init.sls index 91fe11d..fc8b2c1 100644 --- a/postgres/server/init.sls +++ b/postgres/server/init.sls @@ -8,8 +8,6 @@ {%- do includes.append('postgres.upstream') %} {%- endif %} -{%- set pkgs = [postgres.pkg] + postgres.pkgs_extra %} - {%- if includes -%} include: @@ -17,6 +15,8 @@ include: {%- endif %} +{%- set pkgs = [postgres.pkg] + postgres.pkgs_extra %} + # Install, configure and start PostgreSQL server postgresql-server: @@ -82,7 +82,7 @@ postgresql-conf: {{ postgres.postgresconf|indent(8) }} - show_changes: True - append_if_not_found: True - - backup: {{ postgres.postgresconf_backup }} + - backup: {{ postgres.config_backup }} - require: - file: postgresql-config-dir - watch_in: @@ -90,9 +90,23 @@ postgresql-conf: {%- endif %} +{%- set pg_hba_path = salt['file.join'](postgres.conf_dir, 'pg_hba.conf') %} + +postgresql-pg_hba-backup: + file.copy: + - name: {{ pg_hba_path ~ postgres.config_backup }} + - source: {{ pg_hba_path }} + - force: True + - user: {{ postgres.user }} + - group: {{ postgres.group }} + - mode: 600 + - onlyif: test -f {{ pg_hba_path }} + - prereq: + - file: postgresql-pg_hba + postgresql-pg_hba: file.managed: - - name: {{ postgres.conf_dir }}/pg_hba.conf + - name: {{ pg_hba_path }} - user: {{ postgres.user }} - group: {{ postgres.group }} - mode: 600 From 7d692958ee59e3acdaa17e1e258bf02e66c7131e Mon Sep 17 00:00:00 2001 From: ek9 Date: Tue, 24 Jan 2017 18:15:46 +0100 Subject: [PATCH 19/42] add example on how to enable extension on a db --- pillar.example | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pillar.example b/pillar.example index 0d294cf..e4b2c52 100644 --- a/pillar.example +++ b/pillar.example @@ -131,6 +131,14 @@ postgres: lc_ctype: 'en_US.UTF-8' lc_collate: 'en_US.UTF-8' tablespace: 'my_space' + # set custom schema + schemas: + public: + owner: 'localUser' + # enable per-db extension + extensions: + uuid-ossp: + schema: 'public' # optional schemas to enable on database schemas: From ea695fa7d68e47431b6b776b9b2094d75e19c291 Mon Sep 17 00:00:00 2001 From: YetAnotherMinion Date: Sun, 5 Feb 2017 13:12:53 -0600 Subject: [PATCH 20/42] Support upstream repo for Fedora 25, 24, and 23 Resolves #143 Add a macro to override base_url for upstream package url using the grain from 3 most recent Fedora distributions --- postgres/codenamemap.yaml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/postgres/codenamemap.yaml b/postgres/codenamemap.yaml index eb98aa1..ab57a8b 100644 --- a/postgres/codenamemap.yaml +++ b/postgres/codenamemap.yaml @@ -34,6 +34,32 @@ {% endmacro %} + +{% macro fedora_codename(name, version, codename=none) %} + {# + Generate lookup dictionary map for Fedora distributions + + name: + distro codename + version: + PostgreSQL release version + codename: + optional grain value if `name` does not match the one returned by + `oscodename` grain + #} + + {# use upstream version if configured #} + {% if repo.use_upstream_repo %} + {% set version = repo.version %} + {% endif %} + +{{ codename|default(name, true) }}: + # PostgreSQL packages are mostly downloaded from `main` repo component + pkg_repo: + baseurl: 'https://download.postgresql.org/pub/repos/yum/{{ version }}/fedora/fedora-$releasever-$basearch' + +{% endmacro %} + ## Debian GNU/Linux {{ debian_codename('wheezy', '9.1') }} {{ debian_codename('jessie', '9.4') }} @@ -51,4 +77,10 @@ {{ debian_codename('wily', '9.4') }} {{ debian_codename('xenial', '9.5') }} +## Fedora +# `oscodename` grain has long distro name +{{ fedora_codename('Fedora-25', '9.5', 'Fedora 25 (Twenty Five)') }} +{{ fedora_codename('Fedora-24', '9.5', 'Fedora 24 (Twenty Four)') }} +{{ fedora_codename('Fedora-23', '9.4', 'Fedora 23 (Twenty Three)') }} + # vim: ft=sls From 4af85124831f165ffd1653294f92a727db4d18d5 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Mon, 13 Feb 2017 16:27:22 +0200 Subject: [PATCH 21/42] Fix issue #141 --- pillar.example | 2 +- postgres/manage.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pillar.example b/pillar.example index f6e0357..37029ce 100644 --- a/pillar.example +++ b/pillar.example @@ -44,7 +44,7 @@ postgres: # Backup extension for configuration files, defaults to ``.bak``. # Set ``False`` to stop creation of backups when config files change. - {%- if 'status.time' in salt.keys() %} + {%- if salt['status.time']|default(none) is callable %} config_backup: ".backup@{{ salt['status.time']('%y-%m-%d_%H:%M:%S') }}" {%- endif %} diff --git a/postgres/manage.sls b/postgres/manage.sls index 66f5865..ab8c703 100644 --- a/postgres/manage.sls +++ b/postgres/manage.sls @@ -1,7 +1,7 @@ {%- from "postgres/map.jinja" import postgres with context -%} {%- from "postgres/macros.jinja" import format_state with context -%} -{%- if not salt.get('postgres.user_create') %} +{%- if salt['postgres.user_create']|default(none) is not callable %} # Salt states for managing PostgreSQL is not available, # need to provision client binaries first From 22446cc53ef186ffb3d0710523b2907075bb5e8f Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Wed, 15 Feb 2017 12:05:56 +0200 Subject: [PATCH 22/42] Resolve #148: workaround `prereq` causing infinite recursion --- postgres/server/init.sls | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/postgres/server/init.sls b/postgres/server/init.sls index fc8b2c1..5610e65 100644 --- a/postgres/server/init.sls +++ b/postgres/server/init.sls @@ -82,7 +82,8 @@ postgresql-conf: {{ postgres.postgresconf|indent(8) }} - show_changes: True - append_if_not_found: True - - backup: {{ postgres.config_backup }} + {#- Detect empty values (none, '') in the config_backup #} + - backup: {{ postgres.config_backup|default(false, true) }} - require: - file: postgresql-config-dir - watch_in: @@ -92,18 +93,6 @@ postgresql-conf: {%- set pg_hba_path = salt['file.join'](postgres.conf_dir, 'pg_hba.conf') %} -postgresql-pg_hba-backup: - file.copy: - - name: {{ pg_hba_path ~ postgres.config_backup }} - - source: {{ pg_hba_path }} - - force: True - - user: {{ postgres.user }} - - group: {{ postgres.group }} - - mode: 600 - - onlyif: test -f {{ pg_hba_path }} - - prereq: - - file: postgresql-pg_hba - postgresql-pg_hba: file.managed: - name: {{ pg_hba_path }} @@ -115,6 +104,14 @@ postgresql-pg_hba: - template: jinja - defaults: acls: {{ postgres.acls }} + {%- if postgres.config_backup %} + # Create the empty file before managing to overcome the limitation of check_cmd + - onlyif: test -f {{ pg_hba_path }} || touch {{ pg_hba_path }} + # Make a local backup before the file modification + - check_cmd: >- + salt-call --local file.copy + {{ pg_hba_path }} {{ pg_hba_path ~ postgres.config_backup }} remove_existing=true + {%- endif %} {%- else %} - replace: False {%- endif %} From f7b321fbbf6a66a05289a263da7d5b085ac9746a Mon Sep 17 00:00:00 2001 From: Frederic Crozat Date: Tue, 21 Feb 2017 22:18:53 +0100 Subject: [PATCH 23/42] Fix pkg_libpq_dev package name for openSUSE / SLE --- postgres/osmap.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index f41a93e..2b38e02 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -93,5 +93,7 @@ RedHat: Suse: pkg: postgresql-server pkg_client: postgresql + pkg_libpq_dev: postgresql + # vim: ft=sls From 7ced4e2aa4b65c9452fcc6b9b99c825974e0a95d Mon Sep 17 00:00:00 2001 From: Nicolas Rodriguez Date: Tue, 20 Jun 2017 23:40:46 +0200 Subject: [PATCH 24/42] Add support for Debian Stretch --- postgres/codenamemap.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/postgres/codenamemap.yaml b/postgres/codenamemap.yaml index ab57a8b..909ccde 100644 --- a/postgres/codenamemap.yaml +++ b/postgres/codenamemap.yaml @@ -63,11 +63,13 @@ ## Debian GNU/Linux {{ debian_codename('wheezy', '9.1') }} {{ debian_codename('jessie', '9.4') }} +{{ debian_codename('stretch', '9.6') }} # `oscodename` grain has long distro name # if `lsb-release` package not installed {{ debian_codename('wheezy', '9.1', 'Debian GNU/Linux 7 (wheezy)') }} {{ debian_codename('jessie', '9.4', 'Debian GNU/Linux 8 (jessie)') }} +{{ debian_codename('stretch', '9.6', 'Debian GNU/Linux 9 (stretch)') }} ## Ubuntu {{ debian_codename('trusty', '9.3') }} From 1548337bbe2fde4bff44ca10a3e94b3915473147 Mon Sep 17 00:00:00 2001 From: Noel McLoughlin Date: Sat, 23 Sep 2017 20:51:51 +0100 Subject: [PATCH 25/42] For arch, postgresql-libs provides psql --- postgres/osmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 2b38e02..a630d1d 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -7,7 +7,7 @@ Arch: prepare_cluster: command: initdb -D /var/lib/postgresql/data test: test -f /var/lib/postgres/data/PG_VERSION - pkg_client: postgresql + pkg_client: postgresql-lib pkg_dev: postgresql Debian: From ca3ffc5e4dabc789fc3274beb9cfe1404210d678 Mon Sep 17 00:00:00 2001 From: Noel McLoughlin Date: Sat, 23 Sep 2017 21:24:56 +0100 Subject: [PATCH 26/42] For Arch, fix path for initdb --- postgres/osmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index a630d1d..517666f 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -5,7 +5,7 @@ Arch: conf_dir: /var/lib/postgres/data prepare_cluster: - command: initdb -D /var/lib/postgresql/data + command: initdb -D /var/lib/postgres/data test: test -f /var/lib/postgres/data/PG_VERSION pkg_client: postgresql-lib pkg_dev: postgresql From a1cc6056ede55d23cc2246f877a92c781b5ce485 Mon Sep 17 00:00:00 2001 From: Noel McLoughlin Date: Sat, 23 Sep 2017 22:25:11 +0100 Subject: [PATCH 27/42] Darwin support --- postgres/osmap.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 2b38e02..9160d61 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -95,5 +95,11 @@ Suse: pkg_client: postgresql pkg_libpq_dev: postgresql +MacOS: + pkg: postgres + prepare_cluster: + user: _postgres + #binpath: /usr/local/bin + # vim: ft=sls From 772302211be3807d515d81c824e63bea7e468ac8 Mon Sep 17 00:00:00 2001 From: Noel McLoughlin Date: Sat, 23 Sep 2017 22:58:29 +0100 Subject: [PATCH 28/42] Darwin support --- postgres/osmap.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 9160d61..602fdb6 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -96,10 +96,13 @@ Suse: pkg_libpq_dev: postgresql MacOS: + service: postgres pkg: postgres + conf_dir: /usr/local/var/postgres prepare_cluster: + command: initdb -D /usr/local/var/postgres/ + test: test -f /usr/local/var/postgres/PG_VERSION user: _postgres - #binpath: /usr/local/bin # vim: ft=sls From 36851d4e837868e060f2275146c62294aa55ed4a Mon Sep 17 00:00:00 2001 From: Noel McLoughlin Date: Sat, 23 Sep 2017 23:21:12 +0100 Subject: [PATCH 29/42] Darwin support --- postgres/osmap.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 602fdb6..4a95413 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -98,6 +98,8 @@ Suse: MacOS: service: postgres pkg: postgres + pkg_client: + pkg_libpq_dev: conf_dir: /usr/local/var/postgres prepare_cluster: command: initdb -D /usr/local/var/postgres/ From 9e81aec024441925bfd657e819bfd334e8946d5b Mon Sep 17 00:00:00 2001 From: Noel McLoughlin Date: Sat, 23 Sep 2017 23:46:16 +0100 Subject: [PATCH 30/42] Darwin support --- postgres/osmap.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 4a95413..f61fd72 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -97,14 +97,17 @@ Suse: MacOS: service: postgres - pkg: postgres + pkg: postgresql pkg_client: pkg_libpq_dev: conf_dir: /usr/local/var/postgres + user: _postgres + group: _postgres prepare_cluster: command: initdb -D /usr/local/var/postgres/ test: test -f /usr/local/var/postgres/PG_VERSION user: _postgres + group: _postgres # vim: ft=sls From 0efd330346b330896f9f2a49627ddc5fe5df1e8b Mon Sep 17 00:00:00 2001 From: Noel McLoughlin Date: Sun, 24 Sep 2017 00:12:00 +0100 Subject: [PATCH 31/42] Darwin support --- postgres/osmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index f61fd72..58e11bf 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -96,7 +96,7 @@ Suse: pkg_libpq_dev: postgresql MacOS: - service: postgres + service: postgresql pkg: postgresql pkg_client: pkg_libpq_dev: From d378b47f9f743c053a7843f9d4de0e5286534435 Mon Sep 17 00:00:00 2001 From: Noel McLoughlin Date: Sun, 24 Sep 2017 00:51:48 +0100 Subject: [PATCH 32/42] Darwin support --- postgres/osmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 517666f..00c5e2a 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -7,7 +7,7 @@ Arch: prepare_cluster: command: initdb -D /var/lib/postgres/data test: test -f /var/lib/postgres/data/PG_VERSION - pkg_client: postgresql-lib + pkg_client: postgresql-libs pkg_dev: postgresql Debian: From 1065e1c67c6757611220846ecaca61d9caae2093 Mon Sep 17 00:00:00 2001 From: Noel McLoughin Date: Mon, 22 Jan 2018 14:06:03 +0000 Subject: [PATCH 33/42] 'env' should be [] not {} in defaults.yml --- postgres/defaults.yaml | 2 +- postgres/server/init.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/postgres/defaults.yaml b/postgres/defaults.yaml index 50f9acc..2fd1622 100644 --- a/postgres/defaults.yaml +++ b/postgres/defaults.yaml @@ -16,7 +16,7 @@ postgres: command: initdb --pgdata=/var/lib/pgsql/data test: test -f /var/lib/pgsql/data/PG_VERSION user: postgres - env: {} + env: [] conf_dir: /var/lib/pgsql/data postgresconf: "" diff --git a/postgres/server/init.sls b/postgres/server/init.sls index 5610e65..d0d60a3 100644 --- a/postgres/server/init.sls +++ b/postgres/server/init.sls @@ -56,7 +56,7 @@ postgresql-cluster-prepared: - name: {{ postgres.prepare_cluster.command }} - cwd: / - runas: {{ postgres.prepare_cluster.user }} - - env: {{ postgres.prepare_cluster.env|default({}) }} + - env: {{ postgres.prepare_cluster.env }} - unless: - {{ postgres.prepare_cluster.test }} - require: From e62d1373514f3c044bb7bc3a1ac929bd3e32ad1e Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Thu, 22 Feb 2018 17:54:52 +0200 Subject: [PATCH 34/42] Do not fail states during mock test mode --- postgres/macros.jinja | 2 ++ postgres/manage.sls | 16 +++++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/postgres/macros.jinja b/postgres/macros.jinja index 3732df6..4eedddb 100644 --- a/postgres/macros.jinja +++ b/postgres/macros.jinja @@ -23,6 +23,8 @@ {{ state }}-{{ name }}: {{ state }}.{{ ensure|default('present') }}: {{- format_kwargs(kwarg) }} + - onchanges: + - test: postgres-reload-modules {%- endmacro %} diff --git a/postgres/manage.sls b/postgres/manage.sls index ab8c703..20a7309 100644 --- a/postgres/manage.sls +++ b/postgres/manage.sls @@ -18,7 +18,7 @@ include: # Ensure that Salt is able to use postgres modules postgres-reload-modules: - test.nop: + test.succeed_with_changes: - reload_modules: True # User states @@ -26,8 +26,6 @@ postgres-reload-modules: {%- for name, user in postgres.users|dictsort() %} {{ format_state(name, 'postgres_user', user) }} - - require: - - test: postgres-reload-modules {%- endfor %} @@ -36,9 +34,8 @@ postgres-reload-modules: {%- for name, tblspace in postgres.tablespaces|dictsort() %} {{ format_state(name, 'postgres_tablespace', tblspace) }} - - require: - - test: postgres-reload-modules {%- if 'owner' in tblspace %} + - require: - postgres_user: postgres_user-{{ tblspace.owner }} {%- endif %} @@ -49,8 +46,9 @@ postgres-reload-modules: {%- for name, db in postgres.databases|dictsort() %} {{ format_state(name, 'postgres_database', db) }} + {%- if 'owner' in db or 'tablespace' in db %} - require: - - test: postgres-reload-modules + {%- endif %} {%- if 'owner' in db %} - postgres_user: postgres_user-{{ db.owner }} {%- endif %} @@ -65,9 +63,8 @@ postgres-reload-modules: {%- for name, schema in postgres.schemas|dictsort() %} {{ format_state(name, 'postgres_schema', schema) }} - - require: - - test: postgres-reload-modules {%- if 'owner' in schema %} + - require: - postgres_user: postgres_user-{{ schema.owner }} {%- endif %} @@ -78,8 +75,9 @@ postgres-reload-modules: {%- for name, extension in postgres.extensions|dictsort() %} {{ format_state(name, 'postgres_extension', extension) }} + {%- if 'maintenance_db' in extension or 'schema' in extension %} - require: - - test: postgres-reload-modules + {%- endif %} {%- if 'maintenance_db' in extension %} - postgres_database: postgres_database-{{ extension.maintenance_db }} {%- endif %} From db9089018469780cdc64b8d4a701065a310f9660 Mon Sep 17 00:00:00 2001 From: Noel McLoughin Date: Sun, 14 Jan 2018 16:12:17 +0000 Subject: [PATCH 35/42] Tidyup alternatives system priority hardcoding --- pillar.example | 4 ++++ postgres/client.sls | 19 ++++++------------- postgres/defaults.yaml | 4 ++++ postgres/osmap.yaml | 6 +++++- postgres/server/init.sls | 21 ++++++--------------- 5 files changed, 25 insertions(+), 29 deletions(-) diff --git a/pillar.example b/pillar.example index 37029ce..47c00ed 100644 --- a/pillar.example +++ b/pillar.example @@ -14,6 +14,10 @@ postgres: - postgresql-contrib - postgresql-plpython + #'Alternatives system' priority incremental. 0 disables feature. + linux: + altpriority: 30 + # Append the lines under this item to your postgresql.conf file. # Pay attention to indent exactly with 4 spaces for all lines. postgresconf: | diff --git a/postgres/client.sls b/postgres/client.sls index c51ca11..a8729d1 100644 --- a/postgres/client.sls +++ b/postgres/client.sls @@ -8,14 +8,11 @@ {%- endfor %} {%- if postgres.use_upstream_repo %} - include: - postgres.upstream - {%- endif %} # Install PostgreSQL client and libraries - postgresql-client-libs: pkg.installed: - pkgs: {{ pkgs }} @@ -25,23 +22,19 @@ postgresql-client-libs: - pkgrepo: postgresql-repo {%- endif %} -{%- if 'bin_dir' in postgres %} - -# Make client binaries available in $PATH - - {%- for bin in postgres.client_bins %} - - {%- set path = salt['file.join'](postgres.bin_dir, bin) %} +# Alternatives system. Make client binaries available in $PATH +{%- if 'bin_dir' in postgres and postgres.linux.altpriority %} + {%- for bin in postgres.client_bins %} + {%- set path = salt['file.join'](postgres.bin_dir, bin) %} {{ bin }}: alternatives.install: - link: {{ salt['file.join']('/usr/bin', bin) }} - path: {{ path }} - - priority: 30 + - priority: {{ postgres.linux.altpriority }} - onlyif: test -f {{ path }} - require: - pkg: postgresql-client-libs - {%- endfor %} - + {%- endfor %} {%- endif %} diff --git a/postgres/defaults.yaml b/postgres/defaults.yaml index 50f9acc..2e0251c 100644 --- a/postgres/defaults.yaml +++ b/postgres/defaults.yaml @@ -41,3 +41,7 @@ postgres: databases: {} schemas: {} extensions: {} + + linux: + #Alternatives system are disabled by a 'altpriority=0' pillar. + altpriority: diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index bf72717..03c31da 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -46,7 +46,11 @@ RedHat: command: initdb --pgdata='{{ data_dir }}' test: test -f '{{ data_dir }}/PG_VERSION' - # Directory containing PostgreSQL client executables + # Alternatives system + linux: + altpriority: 30 + + # directory containing PostgreSQL client executables bin_dir: /usr/pgsql-{{ repo.version }}/bin client_bins: - clusterdb diff --git a/postgres/server/init.sls b/postgres/server/init.sls index 5610e65..87cd005 100644 --- a/postgres/server/init.sls +++ b/postgres/server/init.sls @@ -9,16 +9,12 @@ {%- endif %} {%- if includes -%} - include: {{ includes|yaml(false)|indent(2) }} - {%- endif %} {%- set pkgs = [postgres.pkg] + postgres.pkgs_extra %} - # Install, configure and start PostgreSQL server - postgresql-server: pkg.installed: - pkgs: {{ pkgs }} @@ -28,27 +24,23 @@ postgresql-server: - pkgrepo: postgresql-repo {%- endif %} -{%- if 'bin_dir' in postgres %} - -# Make server binaries available in $PATH - - {%- for bin in postgres.server_bins %} - - {%- set path = salt['file.join'](postgres.bin_dir, bin) %} +# Alternatives system. Make server binaries available in $PATH +{%- if 'bin_dir' in postgres and postgres.linux.altpriority %} + {%- for bin in postgres.server_bins %} + {%- set path = salt['file.join'](postgres.bin_dir, bin) %} {{ bin }}: alternatives.install: - link: {{ salt['file.join']('/usr/bin', bin) }} - path: {{ path }} - - priority: 30 + - priority: {{ postgres.linux.altpriority }} - onlyif: test -f {{ path }} - require: - pkg: postgresql-server - require_in: - cmd: postgresql-cluster-prepared - {%- endfor %} - + {%- endfor %} {%- endif %} postgresql-cluster-prepared: @@ -138,7 +130,6 @@ postgresql-tablespace-dir-{{ name }}: {%- if not postgres.bake_image %} # Start PostgreSQL server using OS init - postgresql-running: service.running: - name: {{ postgres.service }} From 22b11fd680b1b7d79a167eb65d8a6aaadc695c93 Mon Sep 17 00:00:00 2001 From: Noel McLoughin Date: Sun, 14 Jan 2018 16:05:12 +0000 Subject: [PATCH 36/42] Support for upstream postgresql.org zypp repo after PR review --- pillar.example | 22 ++++++++++------------ postgres/osmap.yaml | 37 +++++++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/pillar.example b/pillar.example index 37029ce..871fea8 100644 --- a/pillar.example +++ b/pillar.example @@ -1,19 +1,20 @@ postgres: - # Set True to configure upstream postgresql.org repository for YUM or APT + # UPSTREAM REPO + # Set True to configure upstream postgresql.org repository for YUM/APT/ZYPP use_upstream_repo: False - # Version to install from upstream repository - version: '9.3' + # Version to install from upstream repository (if upstream_repo: True) + version: '9.6' - # These are Debian/Ubuntu specific package names - pkg: 'postgresql-9.3' - pkg_client: 'postgresql-client-9.3' - - # Additional packages to install with PostgreSQL server, - # this should be in a list format + # PACKAGE + # These pillars are typically never required. + # pkg: 'postgresql' + # pkg_client: 'postgresql-client' + # service: postgresql pkgs_extra: - postgresql-contrib - postgresql-plpython + # POSTGRES # Append the lines under this item to your postgresql.conf file. # Pay attention to indent exactly with 4 spaces for all lines. postgresconf: | @@ -48,9 +49,6 @@ postgres: config_backup: ".backup@{{ salt['status.time']('%y-%m-%d_%H:%M:%S') }}" {%- endif %} - # PostgreSQL service name - service: postgresql - {%- if grains['init'] == 'unknown' %} # If Salt is unable to detect init system running in the scope of state run, diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index bf72717..9a2d17c 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -91,9 +91,43 @@ RedHat: {% endif %} Suse: + pkg_repo: + name: pgdg-sles-{{ release }} + humanname: PostgreSQL {{ repo.version }} $releasever - $basearch + #Using sles-12 upstream repo for opensuse + baseurl: 'https://download.postgresql.org/pub/repos/zypp/{{ repo.version }}/suse/sles-12-$basearch' + key_url: 'https://download.postgresql.org/pub/repos/zypp/{{ repo.version }}/suse/sles-12-$basearch/repodata/repomd.xml.key' + gpgcheck: 1 + gpgautoimport: True + +{% if repo.use_upstream_repo %} + {# Pillars needed are 'use_upstream_repo: True' and 'version: n.n'. #} + {# Avoid setting package names as pillars, as may corrupt postgres. #} + {% set lib_dir = '/var/lib/pgsql/' ~ repo.version ~ '/data' %} + + pkg: postgresql{{ release }}-server + pkg_client: postgresql{{ release }} + conf_dir: {{ lib_dir }} + service: postgresql-{{ repo.version }} + + #This is postgresql-libs in defaults.yml but upstream is libpqxx + pkg_libpq_dev: libpqxx + pkg_dev: postgresql{{ release }}-devel + pkg_libs: postgresql{{ release }}-libs + + prepare_cluster: + #Full path needed as initdb is NOT 'cross version compatible' binary + command: /usr/pgsql-{{ repo.version }}/bin/initdb --pgdata='{{ lib_dir }}' + test: test -f '{{ lib_dir }}/PG_VERSION' + +{% else %} + pkg: postgresql-server pkg_client: postgresql - pkg_libpq_dev: postgresql + pkg_libpq_dev: libqpxx + pkg_libs: postgresql-libs + +{% endif %} MacOS: service: postgresql @@ -109,5 +143,4 @@ MacOS: user: _postgres group: _postgres - # vim: ft=sls From 1e527c4b910738f963e53d2a44c0d8cb77b6f389 Mon Sep 17 00:00:00 2001 From: noelmcloughlin Date: Thu, 8 Mar 2018 11:41:19 +0000 Subject: [PATCH 37/42] POSTGRESAPP on MacOS --- README.rst | 14 +++--- pillar.example | 10 +++- postgres/defaults.yaml | 17 +++++++ postgres/dev.sls | 54 ++++++++++++++++++-- postgres/init.sls | 5 ++ postgres/macos/init.sls | 10 ++++ postgres/macos/postgresapp.sls | 67 +++++++++++++++++++++++++ postgres/osmap.yaml | 22 +++++--- postgres/repo.yaml | 8 +++ postgres/templates/limit.maxfiles.plist | 21 ++++++++ postgres/templates/mac_shortcut.sh | 8 +++ 11 files changed, 218 insertions(+), 18 deletions(-) create mode 100644 postgres/macos/init.sls create mode 100644 postgres/macos/postgresapp.sls create mode 100644 postgres/templates/limit.maxfiles.plist create mode 100755 postgres/templates/mac_shortcut.sh diff --git a/README.rst b/README.rst index bd90d90..01ccb04 100644 --- a/README.rst +++ b/README.rst @@ -17,12 +17,12 @@ Available states ------------ Installs and configures both PostgreSQL server and client with creation of various DB objects in -the cluster. +the cluster. This state applies to both Linux and MacOS. ``postgres.client`` ------------------- -Installs the PostgreSQL client binaries and libraries. +Installs the PostgreSQL client binaries and libraries on Linux. ``postgres.manage`` ------------------- @@ -33,18 +33,18 @@ See ``pillar.example`` file for details. ``postgres.python`` ------------------- -Installs the PostgreSQL adapter for Python. +Installs the PostgreSQL adapter for Python on Linux. ``postgres.server`` ------------------- -Installs the PostgreSQL server package, prepares the DB cluster and starts the server using +Installs the PostgreSQL server package on Linux, prepares the DB cluster and starts the server using packaged init script, job or unit. ``postgres.server.image`` ------------------------- -Installs the PostgreSQL server package, prepares the DB cluster and starts the server by issuing +Installs the PostgreSQL server package on Linux, prepares the DB cluster and starts the server by issuing raw ``pg_ctl`` command. The ``postgres:bake_image`` Pillar toggles this behaviour. For example: .. code:: yaml @@ -77,12 +77,14 @@ The state relies on the ``postgres:use_upstream_repo`` Pillar value which could * ``True`` (default): adds the upstream repository to install packages from * ``False``: makes sure that the repository configuration is absent +* ``postgresapp`` (MacOS) uses upstream PostgresApp package repository. The ``postgres:version`` Pillar controls which version of the PostgreSQL packages should be -installed from the upstream repository. Defaults to ``9.5``. +installed from the upstream Linux repository. Defaults to ``9.5``. Testing ======= +The postgres state was tested on MacOS (El Capitan 10.11.6) Testing is done with the ``kitchen-salt``. diff --git a/pillar.example b/pillar.example index e87a238..338c017 100644 --- a/pillar.example +++ b/pillar.example @@ -5,6 +5,10 @@ postgres: # Version to install from upstream repository (if upstream_repo: True) version: '9.6' + ### MACOS + # Set to 'postgresapp' to install that upstream MacOS package + #use_upstream_repo: 'postgresapp' + # PACKAGE # These pillars are typically never required. # pkg: 'postgresql' @@ -14,11 +18,15 @@ postgres: - postgresql-contrib - postgresql-plpython - #'Alternatives system' priority incremental. 0 disables feature. linux: altpriority: 30 + # macos limits + limits: + soft: 64000 + hard: 64000 + # POSTGRES # Append the lines under this item to your postgresql.conf file. # Pay attention to indent exactly with 4 spaces for all lines. diff --git a/postgres/defaults.yaml b/postgres/defaults.yaml index 50b12b8..4a21fa9 100644 --- a/postgres/defaults.yaml +++ b/postgres/defaults.yaml @@ -9,6 +9,8 @@ postgres: pkg_dev: postgresql-devel pkg_libpq_dev: postgresql-libs python: python-psycopg2 + userhomes: /home + systemuser: user: postgres group: postgres @@ -21,6 +23,21 @@ postgres: conf_dir: /var/lib/pgsql/data postgresconf: "" + macos: + archive: postgres.dmg + tmpdir: /tmp/postgrestmp + postgresapp: + #See: https://github.com/PostgresApp/PostgresApp/releases/ + url: https://github.com/PostgresApp/PostgresApp/releases/download/v2.1.1/Postgres-2.1.1.dmg + sum: sha256=ac0656b522a58fd337931313f09509c09610c4a6078fe0b8e469e69af1e1750b + homebrew: + url: + sum: + dl: + opts: -s -L + interval: 60 + retries: 2 + pg_hba.conf: salt://postgres/templates/pg_hba.conf.j2 acls: # "local" is for Unix domain socket connections only diff --git a/postgres/dev.sls b/postgres/dev.sls index 451d478..3e3d1f1 100644 --- a/postgres/dev.sls +++ b/postgres/dev.sls @@ -1,13 +1,61 @@ {% from "postgres/map.jinja" import postgres with context %} -{% if postgres.pkg_dev %} +{% if grains.os not in ('Windows', 'MacOS',) %} + + {% if postgres.pkg_dev %} install-postgres-dev-package: pkg.installed: - name: {{ postgres.pkg_dev }} -{% endif %} + {% endif %} -{% if postgres.pkg_libpq_dev %} + {% if postgres.pkg_libpq_dev %} install-postgres-libpq-dev: pkg.installed: - name: {{ postgres.pkg_libpq_dev }} + {% endif %} + +{% endif %} + + +{% if grains.os == 'MacOS' %} + + # Darwin maxfiles limits + {% if postgres.limits.soft or postgres.limits.hard %} + +postgres_maxfiles_limits_conf: + file.managed: + - name: /Library/LaunchDaemons/limit.maxfiles.plist + - source: salt://postgres/templates/limit.maxfiles.plist + - context: + soft_limit: {{ postgres.limits.soft or postgres.limits.hard }} + hard_limit: {{ postgres.limits.hard or postgres.limits.soft }} + - group: wheel + {% endif %} + + # MacOS Shortcut for system user + {% if postgres.systemuser|lower not in (None, '',) %} + +postgres-desktop-shortcut-clean: + file.absent: + - name: '{{ postgres.userhomes }}/{{ postgres.systemuser }}/Desktop/postgres' + - require_in: + - file: postgres-desktop-shortcut-add + + {% endif %} + +postgres-desktop-shortcut-add: + file.managed: + - name: /tmp/mac_shortcut.sh + - source: salt://postgres/templates/mac_shortcut.sh + - mode: 755 + - template: jinja + - context: + user: {{ postgres.systemuser }} + homes: {{ postgres.userhomes }} + cmd.run: + - name: /tmp/mac_shortcut.sh {{ postgres.use_upstream_repo }} + - runas: {{ postgres.systemuser }} + - require: + - file: postgres-desktop-shortcut-add + {% endif %} diff --git a/postgres/init.sls b/postgres/init.sls index 0409e62..a28a0ed 100644 --- a/postgres/init.sls +++ b/postgres/init.sls @@ -1,4 +1,9 @@ + include: +{% if grains.os == 'MacOS' %} + - postgres.macos +{% else %} - postgres.server - postgres.client - postgres.manage +{% endif %} diff --git a/postgres/macos/init.sls b/postgres/macos/init.sls new file mode 100644 index 0000000..7ebb4ef --- /dev/null +++ b/postgres/macos/init.sls @@ -0,0 +1,10 @@ +{% from "postgres/map.jinja" import postgres with context %} + +include: +{% if postgres.use_upstream_repo == 'postgresapp' %} + - postgres.macos.postgresapp +{% else %} + - postgres.server + - postgres.client +{% endif %} + - postgres.dev diff --git a/postgres/macos/postgresapp.sls b/postgres/macos/postgresapp.sls new file mode 100644 index 0000000..6f5bd4c --- /dev/null +++ b/postgres/macos/postgresapp.sls @@ -0,0 +1,67 @@ +{% from "postgres/map.jinja" import postgres as pg with context %} + +# Cleanup first +pg-remove-prev-archive: + file.absent: + - name: '{{ pg.macos.tmpdir }}/{{ pg.macos.archive }}' + - require_in: + - pg-extract-dirs + +pg-extract-dirs: + file.directory: + - names: + - '{{ pg.macos.tmpdir }}' + - makedirs: True + - clean: True + - require_in: + - pg-download-archive + +pg-download-archive: + pkg.installed: + - name: curl + cmd.run: + - name: curl {{ pg.macos.dl.opts }} -o '{{ pg.macos.tmpdir }}/{{ pg.macos.archive }}' {{ pg.macos.postgresapp.url }} + {% if grains['saltversioninfo'] >= [2017, 7, 0] %} + - retry: + attempts: {{ pg.macos.dl.retries }} + interval: {{ pg.macos.dl.interval }} + {% endif %} + +{%- if pg.macos.postgresapp.sum %} + #Check hashstring for archive downloads + {%- if grains['saltversioninfo'] <= [2016, 11, 6] %} +pg-check-archive-hash: + module.run: + - name: file.check_hash + - path: '{{ pg.macos.tmpdir }}/{{ pg.macos.archive }}' + - file_hash: {{ pg.macos.postgresapp.sum }} + - onchanges: + - cmd: pg-download-archive + - require_in: + - archive: pg-package-install + {%- endif %} +{%- endif %} + +pg-package-install: + macpackage.installed: + - name: '{{ pg.macos.tmpdir }}/{{ pg.macos.archive }}' + - store: True + - dmg: True + - app: True + - force: True + - allow_untrusted: True + - onchanges: + - cmd: pg-download-archive + - require_in: + - file: pg-package-install + - file: pg-remove-archive + file.append: + - name: {{ pg.userhomes }}/{{ pg.systemuser }}/.bash_profile + - text: 'export PATH=$PATH:/Applications/Postgres.app/Contents/Versions/latest/bin' + +pg-remove-archive: + file.absent: + - name: '{{ pg.macos.tmpdir }}' + - onchanges: + - macpackage: pg-package-install + diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 4cbd9dd..2e1abe8 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -134,17 +134,23 @@ Suse: {% endif %} MacOS: - service: postgresql + {# todo: homebrew postgresql #} + {% if repo.use_upstream_repo == 'homebrew' %} + service: homebrew.mxcl.postgresql + {% elif repo.use_upstream_repo == 'postgresapp' %} + service: com.postgresapp.Postgres2 + {% endif %} pkg: postgresql pkg_client: pkg_libpq_dev: - conf_dir: /usr/local/var/postgres - user: _postgres - group: _postgres + userhomes: /Users + user: {{ repo.user }} + group: {{ repo.group }} + conf_dir: /Users/{{ repo.user }}/Library/AppSupport/postgres_{{ repo.use_upstream_repo }} prepare_cluster: - command: initdb -D /usr/local/var/postgres/ - test: test -f /usr/local/var/postgres/PG_VERSION - user: _postgres - group: _postgres + command: initdb -D /Users/{{ repo.user }}/Library/AppSupport/postgres_{{ repo.use_upstream_repo }} + test: test -f /Users/{{ repo.user }}/Library/AppSupport/postgres_{{ repo.use_upstream_repo }}/PG_VERSION + user: {{ repo.user }} + group: {{ repo.group }} # vim: ft=sls diff --git a/postgres/repo.yaml b/postgres/repo.yaml index e37eec7..1d87463 100644 --- a/postgres/repo.yaml +++ b/postgres/repo.yaml @@ -8,4 +8,12 @@ use_upstream_repo: {{ salt['pillar.get']('postgres:use_upstream_repo', version: {{ salt['pillar.get']('postgres:version', defaults.postgres.version) }} +#Early lookup for system user on MacOS +{% if grains.os == 'MacOS' %} + {% set sysuser = salt['pillar.get']('postgres.user') or salt['cmd.run']("stat -f '%Su' /dev/console") %} + {% set sysgroup = salt['pillar.get']('postgres.group') or salt['cmd.run']("stat -f '%Sg' /dev/console") %} +user: {{ sysuser }} +group: {{ sysgroup }} +{% endif %} + # vim: ft=sls diff --git a/postgres/templates/limit.maxfiles.plist b/postgres/templates/limit.maxfiles.plist new file mode 100644 index 0000000..84a6437 --- /dev/null +++ b/postgres/templates/limit.maxfiles.plist @@ -0,0 +1,21 @@ + + + + + Label + limit.maxfiles + ProgramArguments + + launchctl + limit + maxfiles + {{ soft_limit }} + {{ hard_limit }} + + RunAtLoad + + ServiceIPC + + + diff --git a/postgres/templates/mac_shortcut.sh b/postgres/templates/mac_shortcut.sh new file mode 100755 index 0000000..a156e24 --- /dev/null +++ b/postgres/templates/mac_shortcut.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +vendor=$1 +app="postgres.app" +Source="/Applications/$app" +Destination="{{ homes }}/{{ user }}/Desktop" +/usr/bin/osascript -e "tell application \"Finder\" to make alias file to POSIX file \"$Source\" at POSIX file \"$Destination\"" + From a34a4c270cf3ddca0b1c7b84735738a54bdacd07 Mon Sep 17 00:00:00 2001 From: noelmcloughlin Date: Thu, 8 Mar 2018 12:30:25 +0000 Subject: [PATCH 38/42] Homebrew Postgres on MacOS --- README.rst | 3 ++- pillar.example | 3 ++- postgres/client.sls | 4 +-- postgres/codenamemap.yaml | 4 +-- postgres/dev.sls | 18 ++++++-------- postgres/macos/init.sls | 2 +- postgres/macos/postgresapp.sls | 7 ++---- postgres/osmap.yaml | 14 ++++++----- postgres/server/init.sls | 33 ++++++++++++++++++++----- postgres/templates/limit.maxfiles.plist | 2 +- postgres/templates/mac_shortcut.sh | 4 +-- postgres/upstream.sls | 4 ++- 12 files changed, 60 insertions(+), 38 deletions(-) diff --git a/README.rst b/README.rst index 01ccb04..f28a0b1 100644 --- a/README.rst +++ b/README.rst @@ -77,7 +77,8 @@ The state relies on the ``postgres:use_upstream_repo`` Pillar value which could * ``True`` (default): adds the upstream repository to install packages from * ``False``: makes sure that the repository configuration is absent -* ``postgresapp`` (MacOS) uses upstream PostgresApp package repository. +* ``'postgresapp'`` (MacOS) uses upstream PostgresApp package repository. +* ``'homebrew'`` (MacOS) uses Homebrew postgres The ``postgres:version`` Pillar controls which version of the PostgreSQL packages should be installed from the upstream Linux repository. Defaults to ``9.5``. diff --git a/pillar.example b/pillar.example index 338c017..38f16e8 100644 --- a/pillar.example +++ b/pillar.example @@ -6,8 +6,9 @@ postgres: version: '9.6' ### MACOS - # Set to 'postgresapp' to install that upstream MacOS package + # Set to 'postgresapp' OR 'homebrew' for MacOS #use_upstream_repo: 'postgresapp' + #use_upstream_repo: 'homebrew' # PACKAGE # These pillars are typically never required. diff --git a/postgres/client.sls b/postgres/client.sls index a8729d1..473f3c6 100644 --- a/postgres/client.sls +++ b/postgres/client.sls @@ -7,7 +7,7 @@ {%- endif %} {%- endfor %} -{%- if postgres.use_upstream_repo %} +{%- if postgres.use_upstream_repo == true %} include: - postgres.upstream {%- endif %} @@ -16,7 +16,7 @@ include: postgresql-client-libs: pkg.installed: - pkgs: {{ pkgs }} -{%- if postgres.use_upstream_repo %} +{%- if postgres.use_upstream_repo == true %} - refresh: True - require: - pkgrepo: postgresql-repo diff --git a/postgres/codenamemap.yaml b/postgres/codenamemap.yaml index 909ccde..f5ea154 100644 --- a/postgres/codenamemap.yaml +++ b/postgres/codenamemap.yaml @@ -16,7 +16,7 @@ #} {# use upstream version if configured #} - {% if repo.use_upstream_repo %} + {% if repo.use_upstream_repo == true %} {% set version = repo.version %} {% endif %} @@ -49,7 +49,7 @@ #} {# use upstream version if configured #} - {% if repo.use_upstream_repo %} + {% if repo.use_upstream_repo == true %} {% set version = repo.version %} {% endif %} diff --git a/postgres/dev.sls b/postgres/dev.sls index 3e3d1f1..c9f330b 100644 --- a/postgres/dev.sls +++ b/postgres/dev.sls @@ -29,20 +29,17 @@ postgres_maxfiles_limits_conf: - context: soft_limit: {{ postgres.limits.soft or postgres.limits.hard }} hard_limit: {{ postgres.limits.hard or postgres.limits.soft }} - - group: wheel + - group: {{ postgres.group }} {% endif %} - # MacOS Shortcut for system user - {% if postgres.systemuser|lower not in (None, '',) %} - + {% if postgres.use_upstream_repo == 'postgresapp' %} + # Shortcut for PostgresApp postgres-desktop-shortcut-clean: file.absent: - - name: '{{ postgres.userhomes }}/{{ postgres.systemuser }}/Desktop/postgres' + - name: '{{ postgres.userhomes }}/{{ postgres.user }}/Desktop/Postgres ({{ postgres.use_upstream_repo }})' - require_in: - file: postgres-desktop-shortcut-add - {% endif %} - postgres-desktop-shortcut-add: file.managed: - name: /tmp/mac_shortcut.sh @@ -50,12 +47,13 @@ postgres-desktop-shortcut-add: - mode: 755 - template: jinja - context: - user: {{ postgres.systemuser }} + user: {{ postgres.user }} homes: {{ postgres.userhomes }} cmd.run: - - name: /tmp/mac_shortcut.sh {{ postgres.use_upstream_repo }} - - runas: {{ postgres.systemuser }} + - name: '/tmp/mac_shortcut.sh "Postgres ({{ postgres.use_upstream_repo }})"' + - runas: {{ postgres.user }} - require: - file: postgres-desktop-shortcut-add + {% endif %} {% endif %} diff --git a/postgres/macos/init.sls b/postgres/macos/init.sls index 7ebb4ef..ed332ad 100644 --- a/postgres/macos/init.sls +++ b/postgres/macos/init.sls @@ -3,7 +3,7 @@ include: {% if postgres.use_upstream_repo == 'postgresapp' %} - postgres.macos.postgresapp -{% else %} +{% elif postgres.use_upstream_repo == 'homebrew' %} - postgres.server - postgres.client {% endif %} diff --git a/postgres/macos/postgresapp.sls b/postgres/macos/postgresapp.sls index 6f5bd4c..cb4cd4f 100644 --- a/postgres/macos/postgresapp.sls +++ b/postgres/macos/postgresapp.sls @@ -27,9 +27,7 @@ pg-download-archive: interval: {{ pg.macos.dl.interval }} {% endif %} -{%- if pg.macos.postgresapp.sum %} - #Check hashstring for archive downloads - {%- if grains['saltversioninfo'] <= [2016, 11, 6] %} + {%- if pg.macos.postgresapp.sum %} pg-check-archive-hash: module.run: - name: file.check_hash @@ -40,7 +38,6 @@ pg-check-archive-hash: - require_in: - archive: pg-package-install {%- endif %} -{%- endif %} pg-package-install: macpackage.installed: @@ -56,7 +53,7 @@ pg-package-install: - file: pg-package-install - file: pg-remove-archive file.append: - - name: {{ pg.userhomes }}/{{ pg.systemuser }}/.bash_profile + - name: {{ pg.userhomes }}/{{ pg.user }}/.bash_profile - text: 'export PATH=$PATH:/Applications/Postgres.app/Contents/Versions/latest/bin' pg-remove-archive: diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 2e1abe8..71c6344 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -33,7 +33,7 @@ RedHat: gpgcheck: 1 gpgkey: 'https://download.postgresql.org/pub/repos/yum/RPM-GPG-KEY-PGDG-{{ release }}' -{% if repo.use_upstream_repo %} +{% if repo.use_upstream_repo == true %} {% set data_dir = '/var/lib/pgsql/' ~ repo.version ~ '/data' %} @@ -104,7 +104,7 @@ Suse: gpgcheck: 1 gpgautoimport: True -{% if repo.use_upstream_repo %} +{% if repo.use_upstream_repo == true %} {# Pillars needed are 'use_upstream_repo: True' and 'version: n.n'. #} {# Avoid setting package names as pillars, as may corrupt postgres. #} {% set lib_dir = '/var/lib/pgsql/' ~ repo.version ~ '/data' %} @@ -133,13 +133,14 @@ Suse: {% endif %} +{%- if grains.os == 'MacOS' %} +## jinja check avoids rendering noise/failure on Linux MacOS: - {# todo: homebrew postgresql #} - {% if repo.use_upstream_repo == 'homebrew' %} + {%- if repo.use_upstream_repo == 'homebrew' %} service: homebrew.mxcl.postgresql - {% elif repo.use_upstream_repo == 'postgresapp' %} + {%- elif repo.use_upstream_repo == 'postgresapp' %} service: com.postgresapp.Postgres2 - {% endif %} + {%- endif %} pkg: postgresql pkg_client: pkg_libpq_dev: @@ -152,5 +153,6 @@ MacOS: test: test -f /Users/{{ repo.user }}/Library/AppSupport/postgres_{{ repo.use_upstream_repo }}/PG_VERSION user: {{ repo.user }} group: {{ repo.group }} +{%- endif %} # vim: ft=sls diff --git a/postgres/server/init.sls b/postgres/server/init.sls index fe71435..3c26f36 100644 --- a/postgres/server/init.sls +++ b/postgres/server/init.sls @@ -4,7 +4,7 @@ {%- if postgres.bake_image %} {%- do includes.append('postgres.server.image') %} {%- endif %} -{%- if postgres.use_upstream_repo -%} +{%- if postgres.use_upstream_repo == true -%} {%- do includes.append('postgres.upstream') %} {%- endif %} @@ -18,16 +18,27 @@ include: postgresql-server: pkg.installed: - pkgs: {{ pkgs }} -{%- if postgres.use_upstream_repo %} +{%- if postgres.use_upstream_repo == true %} - refresh: True - require: - pkgrepo: postgresql-repo {%- endif %} + {%- if grains.os == 'MacOS' %} + #Register as Launchd LaunchAgent for system users + - require_in: + - file: postgresql-server + file.managed: + - name: /Library/LaunchAgents/{{ postgres.service }}.plist + - source: /usr/local/opt/postgres/{{ postgres.service }}.plist + - group: wheel + - require_in: + - service: postgresql-running + {%- else %} # Alternatives system. Make server binaries available in $PATH -{%- if 'bin_dir' in postgres and postgres.linux.altpriority %} - {%- for bin in postgres.server_bins %} - {%- set path = salt['file.join'](postgres.bin_dir, bin) %} + {%- if 'bin_dir' in postgres and postgres.linux.altpriority %} + {%- for bin in postgres.server_bins %} + {%- set path = salt['file.join'](postgres.bin_dir, bin) %} {{ bin }}: alternatives.install: @@ -40,7 +51,9 @@ postgresql-server: - require_in: - cmd: postgresql-cluster-prepared - {%- endfor %} + {%- endfor %} + {%- endif %} + {%- endif %} postgresql-cluster-prepared: @@ -59,6 +72,12 @@ postgresql-config-dir: - name: {{ postgres.conf_dir }} - user: {{ postgres.user }} - group: {{ postgres.group }} + - dir_mode: 775 + - force: True + - file_mode: 644 + - recurse: + - user + - group - makedirs: True - require: - cmd: postgresql-cluster-prepared @@ -134,7 +153,9 @@ postgresql-running: service.running: - name: {{ postgres.service }} - enable: True + {% if grains.os not in ('MacOS',) %} - reload: True + {% endif %} - watch: - file: postgresql-pg_hba diff --git a/postgres/templates/limit.maxfiles.plist b/postgres/templates/limit.maxfiles.plist index 84a6437..1b16b1b 100644 --- a/postgres/templates/limit.maxfiles.plist +++ b/postgres/templates/limit.maxfiles.plist @@ -7,7 +7,7 @@ limit.maxfiles ProgramArguments - launchctl + /bin/launchctl limit maxfiles {{ soft_limit }} diff --git a/postgres/templates/mac_shortcut.sh b/postgres/templates/mac_shortcut.sh index a156e24..4f560bf 100755 --- a/postgres/templates/mac_shortcut.sh +++ b/postgres/templates/mac_shortcut.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -vendor=$1 +shortcutName='${1}' app="postgres.app" Source="/Applications/$app" -Destination="{{ homes }}/{{ user }}/Desktop" +Destination="{{ homes }}/{{ user }}/Desktop/${shortcutName}" /usr/bin/osascript -e "tell application \"Finder\" to make alias file to POSIX file \"$Source\" at POSIX file \"$Destination\"" diff --git a/postgres/upstream.sls b/postgres/upstream.sls index 6c3956f..cf92872 100644 --- a/postgres/upstream.sls +++ b/postgres/upstream.sls @@ -3,7 +3,7 @@ {%- if 'pkg_repo' in postgres -%} - {%- if postgres.use_upstream_repo -%} + {%- if postgres.use_upstream_repo == true -%} # Add upstream repository for your distro postgresql-repo: @@ -25,9 +25,11 @@ postgresql-repo: {%- else -%} # Notify that we don't manage this distro + {% if grains.os not in ('Windows', 'MacOS',) %} postgresql-repo: test.show_notification: - text: | PostgreSQL does not provide package repository for {{ grains['osfinger'] }} + {% endif %} {%- endif %} From 60cc787b16da6a8e42bd1cdb3b2d2a3668f3258e Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Tue, 16 Jan 2018 19:17:44 +0100 Subject: [PATCH 39/42] Add support for configure pg_ident.conf --- postgres/defaults.yaml | 3 ++ postgres/server/init.sls | 28 ++++++++++++++++ postgres/templates/pg_ident.conf.j2 | 51 +++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 postgres/templates/pg_ident.conf.j2 diff --git a/postgres/defaults.yaml b/postgres/defaults.yaml index 4a21fa9..4d575cc 100644 --- a/postgres/defaults.yaml +++ b/postgres/defaults.yaml @@ -47,6 +47,9 @@ postgres: # IPv6 local connections: - ['host', 'all', 'all', '::1/128', 'md5'] + pg_ident.conf: salt://postgres/templates/pg_ident.conf.j2 + identity_map: [] + config_backup: '.bak' service: postgresql diff --git a/postgres/server/init.sls b/postgres/server/init.sls index 3c26f36..6e2b23b 100644 --- a/postgres/server/init.sls +++ b/postgres/server/init.sls @@ -129,6 +129,33 @@ postgresql-pg_hba: - require: - file: postgresql-config-dir +{%- set pg_ident_path = salt['file.join'](postgres.conf_dir, 'pg_ident.conf') %} + +postgresql-pg_ident: + file.managed: + - name: {{ pg_ident_path }} + - user: {{ postgres.user }} + - group: {{ postgres.group }} + - mode: 600 +{%- if postgres.identity_map %} + - source: {{ postgres['pg_ident.conf'] }} + - template: jinja + - defaults: + mappings: {{ postgres.identity_map }} + {%- if postgres.config_backup %} + # Create the empty file before managing to overcome the limitation of check_cmd + - onlyif: test -f {{ pg_ident_path }} || touch {{ pg_ident_path }} + # Make a local backup before the file modification + - check_cmd: >- + salt-call --local file.copy + {{ pg_ident_path }} {{ pg_ident_path ~ postgres.config_backup }} remove_existing=true + {%- endif %} +{%- else %} + - replace: False +{%- endif %} + - require: + - file: postgresql-config-dir + {%- for name, tblspace in postgres.tablespaces|dictsort() %} postgresql-tablespace-dir-{{ name }}: @@ -158,5 +185,6 @@ postgresql-running: {% endif %} - watch: - file: postgresql-pg_hba + - file: postgresql-pg_ident {%- endif %} diff --git a/postgres/templates/pg_ident.conf.j2 b/postgres/templates/pg_ident.conf.j2 new file mode 100644 index 0000000..1d4696b --- /dev/null +++ b/postgres/templates/pg_ident.conf.j2 @@ -0,0 +1,51 @@ +###################################################################### +# ATTENTION! Managed by SaltStack. # +# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN! # +###################################################################### +# +# PostgreSQL User Name Maps +# ========================= +# +# Refer to the PostgreSQL documentation, chapter "Client +# Authentication" for a complete description. A short synopsis +# follows. +# +# This file controls PostgreSQL user name mapping. It maps external +# user names to their corresponding PostgreSQL user names. Records +# are of the form: +# +# MAPNAME SYSTEM-USERNAME PG-USERNAME +# +# (The uppercase quantities must be replaced by actual values.) +# +# MAPNAME is the (otherwise freely chosen) map name that was used in +# pg_hba.conf. SYSTEM-USERNAME is the detected user name of the +# client. PG-USERNAME is the requested PostgreSQL user name. The +# existence of a record specifies that SYSTEM-USERNAME may connect as +# PG-USERNAME. +# +# If SYSTEM-USERNAME starts with a slash (/), it will be treated as a +# regular expression. Optionally this can contain a capture (a +# parenthesized subexpression). The substring matching the capture +# will be substituted for \1 (backslash-one) if present in +# PG-USERNAME. +# +# Multiple maps may be specified in this file and used by pg_hba.conf. +# +# No map names are defined in the default configuration. If all +# system user names and PostgreSQL user names are the same, you don't +# need anything in this file. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can +# use "pg_ctl reload" to do that. + +# Put your actual configuration here +# ---------------------------------- + +# MAPNAME SYSTEM-USERNAME PG-USERNAME + +{%- for mapping in mappings %} +{{ '{0:<15} {1:<22} {2}'.format(mapping) -}} +{% endfor %} From f048ab87cd788e3a6a89c2a6cb4a4400b3d21d9c Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 12 Mar 2018 16:48:56 +0100 Subject: [PATCH 40/42] Add example usage of identity mapping --- pillar.example | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pillar.example b/pillar.example index 38f16e8..fe29f4d 100644 --- a/pillar.example +++ b/pillar.example @@ -54,9 +54,14 @@ postgres: # If ``acls`` item value is empty ('', [], null), then the contents of # ``pg_hba.conf`` file will not be touched at all. acls: + - ['local', 'db0', 'connuser', 'peer map=users_as_appuser'] - ['local', 'db1', 'localUser'] - ['host', 'db2', 'remoteUser', '192.168.33.0/24'] + identity_map: + - ['users_as_appuser', 'jdoe', 'connuser'] + - ['users_as_appuser', 'jsmith', 'connuser'] + # Backup extension for configuration files, defaults to ``.bak``. # Set ``False`` to stop creation of backups when config files change. {%- if salt['status.time']|default(none) is callable %} From 7abdb58ac7e06abf6cc750be383b7fc7f02ba405 Mon Sep 17 00:00:00 2001 From: noelmcloughlin Date: Sat, 17 Mar 2018 01:03:10 +0000 Subject: [PATCH 41/42] Fix suse alternatives #189 --- postgres/osmap.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 71c6344..ccf4aa2 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -124,6 +124,33 @@ Suse: command: /usr/pgsql-{{ repo.version }}/bin/initdb --pgdata='{{ lib_dir }}' test: test -f '{{ lib_dir }}/PG_VERSION' + # Alternatives system + linux: + altpriority: 30 + + # directory containing PostgreSQL client executables + bin_dir: /usr/pgsql-{{ repo.version }}/bin + client_bins: + - pg_archivecleanup + - pg_config + - pg_isready + - pg_receivexlog + - pg_rewind + - pg_test_fsync + - pg_test_timing + - pg_upgrade + - pg_xlogdump + - pgbench + server_bins: + - initdb + - pg_controldata + - pg_ctl + - pg_resetxlog + - postgres + - postgresql{{ release }}-check-db-dir + - postgresql{{ release }}-setup + - postmaster + {% else %} pkg: postgresql-server From 4d629f0b0cc70b40a0720f6fac338e250398bf09 Mon Sep 17 00:00:00 2001 From: noelmcloughlin Date: Tue, 20 Mar 2018 14:15:44 +0000 Subject: [PATCH 42/42] corrections for packages --- postgres/defaults.yaml | 6 ++- postgres/osmap.yaml | 84 ++++++++++++++++++++++++++++++++++++++++-- postgres/python.sls | 2 +- 3 files changed, 85 insertions(+), 7 deletions(-) diff --git a/postgres/defaults.yaml b/postgres/defaults.yaml index 50f9acc..e104796 100644 --- a/postgres/defaults.yaml +++ b/postgres/defaults.yaml @@ -7,8 +7,10 @@ postgres: pkgs_extra: [] pkg_client: postgresql-client pkg_dev: postgresql-devel - pkg_libpq_dev: postgresql-libs - python: python-psycopg2 + pkg_libpq_dev: libpq-dev + pkg_libs: postgresql-libs + pkg_python: python-psycopg2 + userhomes: /home user: postgres group: postgres diff --git a/postgres/osmap.yaml b/postgres/osmap.yaml index 00c5e2a..ba13eba 100644 --- a/postgres/osmap.yaml +++ b/postgres/osmap.yaml @@ -17,7 +17,6 @@ Debian: file: /etc/apt/sources.list.d/pgdg.list pkg_repo_keyid: ACCC4CF8 pkg_dev: postgresql-server-dev-all - pkg_libpq_dev: libpq-dev FreeBSD: user: pgsql @@ -33,12 +32,13 @@ RedHat: gpgcheck: 1 gpgkey: 'https://download.postgresql.org/pub/repos/yum/RPM-GPG-KEY-PGDG-{{ release }}' -{% if repo.use_upstream_repo %} - +{% if repo.use_upstream_repo == true %} {% set data_dir = '/var/lib/pgsql/' ~ repo.version ~ '/data' %} pkg: postgresql{{ release }}-server pkg_client: postgresql{{ release }} + pkg_libs: postgresql{{ release }}-libs + pkg_dev: postgresql{{ release }}-devel conf_dir: /var/lib/pgsql/{{ repo.version }}/data service: postgresql-{{ repo.version }} @@ -89,11 +89,87 @@ RedHat: pkg_client: postgresql {% endif %} + pkg_libpq_dev: libpqxx-devel Suse: + pkg_repo: + name: pgdg-sles-{{ release }} + humanname: PostgreSQL {{ repo.version }} $releasever - $basearch + #Using sles-12 upstream repo for opensuse + baseurl: 'https://download.postgresql.org/pub/repos/zypp/{{ repo.version }}/suse/sles-12-$basearch' + key_url: 'https://download.postgresql.org/pub/repos/zypp/{{ repo.version }}/suse/sles-12-$basearch/repodata/repomd.xml.key' + gpgcheck: 1 + gpgautoimport: True + +{% if repo.use_upstream_repo == true %} + {% set lib_dir = '/var/lib/pgsql/' ~ repo.version ~ '/data' %} + + pkg: postgresql{{ release }}-server + pkg_client: postgresql{{ release }} + pkg_dev: postgresql{{ release }}-devel + pkg_libs: postgresql{{ release }}-libs + conf_dir: {{ lib_dir }} + service: postgresql-{{ repo.version }} + + prepare_cluster: + command: /usr/pgsql-{{ repo.version }}/bin/initdb --pgdata='{{ lib_dir }}' + test: test -f '{{ lib_dir }}/PG_VERSION' + + # Alternatives system + linux: + altpriority: 30 + + # directory containing PostgreSQL client executables + bin_dir: /usr/pgsql-{{ repo.version }}/bin + client_bins: + - pg_archivecleanup + - pg_config + - pg_isready + - pg_receivexlog + - pg_rewind + - pg_test_fsync + - pg_test_timing + - pg_upgrade + - pg_xlogdump + - pgbench + server_bins: + - initdb + - pg_controldata + - pg_ctl + - pg_resetxlog + - postgres + - postgresql{{ release }}-check-db-dir + - postgresql{{ release }}-setup + - postmaster + +{% else %} + pkg: postgresql-server pkg_client: postgresql - pkg_libpq_dev: postgresql +{% endif %} + pkg_libpq_dev: libqpxx + +{%- if grains.os == 'MacOS' %} +## jinja check avoids rendering noise/failure on Linux +MacOS: + {%- if repo.use_upstream_repo == 'homebrew' %} + service: homebrew.mxcl.postgresql + {%- elif repo.use_upstream_repo == 'postgresapp' %} + service: com.postgresapp.Postgres2 + {%- endif %} + pkg: postgresql + pkg_client: + pkg_libpq_dev: + userhomes: /Users + user: {{ repo.user }} + group: {{ repo.group }} + conf_dir: /Users/{{ repo.user }}/Library/AppSupport/postgres_{{ repo.use_upstream_repo }} + prepare_cluster: + command: initdb -D /Users/{{ repo.user }}/Library/AppSupport/postgres_{{ repo.use_upstream_repo }} + test: test -f /Users/{{ repo.user }}/Library/AppSupport/postgres_{{ repo.use_upstream_repo }}/PG_VERSION + user: {{ repo.user }} + group: {{ repo.group }} +{%- endif %} # vim: ft=sls diff --git a/postgres/python.sls b/postgres/python.sls index a35d721..a206bfc 100644 --- a/postgres/python.sls +++ b/postgres/python.sls @@ -2,4 +2,4 @@ postgresql-python: pkg.installed: - - name: {{ postgres.python}} + - name: {{ postgres.pkg_python}}