SmartOS esky packaging of 2014.7

Resulting builds can be used either with zeromq or with raet

pkg/smartos/esky/BUILD_ENVIRONMENT.md

@@ -13,18 +13,18 @@ HERE=$(pwd)
 
 mv /opt/local /opt/local.backup ; hash -r
 cd /
-curl http://pkgsrc.joyent.com/packages/SmartOS/bootstrap/bootstrap-2013Q4-x86_64.tar.gz | gtar xz
+curl http://pkgsrc.joyent.com/packages/SmartOS/bootstrap/bootstrap-2014Q2-x86_64.tar.gz | gtar xz
 hash -r
 
 pkgin -y up
-pkgin -y in build-essential salt swig py27-pip unzip py27-mysqldb
-pkgin -y rm salt
+pkgin -y in build-essential salt swig py27-pip unzip py27-mysqldb libsodium
+pkgin -y rm salt py27-zmq
 
 cd /opt/local/bin
 curl -kO 'https://us-east.manta.joyent.com/nahamu/public/smartos/bins/patchelf'
 chmod +x patchelf
 
-pip install esky bbfreeze
+pip install --egg esky bbfreeze
 
 cd $HERE
 curl -kO 'https://pypi.python.org/packages/source/b/bbfreeze-loader/bbfreeze-loader-1.1.0.zip'
@@ -38,11 +38,20 @@ patchelf --set-rpath '$ORIGIN:$ORIGIN/../lib' $HERE/console.exe
 
 find /opt/local -name console.exe -exec mv $HERE/console.exe {} \;
 
-git clone git://github.com/saltstack/salt -b 2014.1
+git clone git://github.com/saltstack/salt -b 2014.7
 cd $HERE/salt
-pip install -r requirements.txt
-# packages not in main requirements file that are nice to have
-pip install -r pkg/smartos/esky/requirements.txt
+
+# install all requirements
+# (installing them as eggs seems to trigger esky pulling in the whole egg)
+# this step is buggy... I had to run them repeatedly until they succeeded...
+pip install --egg -r pkg/smartos/esky/zeromq_requirements.txt
+pip install --egg -r pkg/smartos/esky/raet_requirements.txt
+
+# install the sodium_grabber library
+python pkg/smartos/esky/sodium_grabber_installer.py install
+
+# at this point you have a build environment that you could set aside and reuse to run further builds.
+
 bash pkg/smartos/esky/build-tarball.sh
 
 # Upload packages into Manta

pkg/smartos/esky/sodium_grabber.c

@@ -0,0 +1,16 @@
+#include <Python.h>
+#include <sodium.h>
+
+static PyObject* grabber_init(PyObject* self, PyObject* args)                                                                                                                                                                                {
+  return Py_BuildValue("i", sodium_init());
+}
+
+PyMethodDef methods[] = {
+  {"init", grabber_init, METH_VARARGS},
+  {NULL, NULL},
+};
+
+void initsodium_grabber()
+{
+  (void)Py_InitModule("sodium_grabber", methods);
+}

pkg/smartos/esky/sodium_grabber_installer.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+'''
+The setup script for sodium_grabber
+'''
+
+# pylint: disable=C0111,E1101,E1103,F0401,W0611
+
+from distutils.core import setup, Extension
+from os import path
+
+HERE = path.dirname(__file__)
+
+SETUP_KWARGS = {}
+sodium_grabber = Extension('sodium_grabber',
+    sources = [path.join(HERE, 'sodium_grabber.c')],
+    libraries = ['sodium'],
+)
+SETUP_KWARGS['ext_modules'] = [sodium_grabber]
+SETUP_KWARGS['name'] = "sodium_grabber"
+
+if __name__ == '__main__':
+    setup(**SETUP_KWARGS)

pkg/smartos/esky/zeromq_requirements.txt

@@ -1,2 +1,7 @@
--r ../../../zeromq-requirements.txt
+# Need to set a specific version of pyzmq, so can't use the main project's requirements file... have to copy it in and modify...
+#-r ../../../zeromq-requirements.txt
+-r ../../../_requirements.txt
+M2Crypto
+pycrypto
+pyzmq == 13.1.0
 -r requirements.txt

setup.py

@@ -683,12 +683,21 @@ elif sys.platform.startswith('sunos'):
         from bbfreeze.modulegraph.modulegraph import ModuleGraph
         mf = ModuleGraph(sys.path[:])
         for arg in glob.glob('salt/modules/*.py'):
-                mf.run_script(arg)
+            mf.run_script(arg)
         for mod in mf.flatten():
             if type(mod).__name__ != 'Script' and mod.filename:
                 FREEZER_INCLUDES.append(str(os.path.basename(mod.identifier)))
     except ImportError:
         pass
+    # Include C extension that convinces esky to package up the libsodium C library
+    # This is needed for ctypes to find it in libnacl which is in turn needed for raet
+    # see pkg/smartos/esky/sodium_grabber{.c,_installer.py}
+    FREEZER_INCLUDES.extend([
+        'sodium_grabber',
+        'ioflo',
+        'raet',
+        'libnacl',
+    ])
 
 if HAS_ESKY:
     # if the user has the esky / bbfreeze libraries installed, add the