Fix spm http py3 install

salt/spm/__init__.py

@@ -4,8 +4,6 @@ This module provides the point of entry to SPM, the Salt Package Manager
 
 .. versionadded:: 2015.8.0
 """
-
-# Import Python libs
 from __future__ import absolute_import, print_function, unicode_literals
 
 import hashlib
@@ -16,8 +14,6 @@ import sys
 import tarfile
 
 import salt.cache
-
-# Import Salt libs
 import salt.client
 import salt.config
 import salt.loader
@@ -373,8 +369,10 @@ class SPMClient(object):
                 dl_url = dl_url.replace("file://", "")
                 shutil.copyfile(dl_url, out_file)
             else:
-                with salt.utils.files.fopen(out_file, "w") as outf:
-                    outf.write(self._query_http(dl_url, repo_info["info"]))
+                with salt.utils.files.fopen(out_file, "wb") as outf:
+                    outf.write(
+                        self._query_http(dl_url, repo_info["info"], decode_body=False)
+                    )
 
         # First we download everything, then we install
         for package in dl_list:
@@ -666,7 +664,7 @@ class SPMClient(object):
                         continue
                     callback(repo, repo_data[repo])
 
-    def _query_http(self, dl_path, repo_info):
+    def _query_http(self, dl_path, repo_info, decode_body=True):
         """
         Download files via http
         """
@@ -682,6 +680,7 @@ class SPMClient(object):
                             text=True,
                             username=repo_info["username"],
                             password=repo_info["password"],
+                            decode_body=decode_body,
                         )
                     else:
                         raise SPMException(
@@ -692,7 +691,7 @@ class SPMClient(object):
                 except SPMException as exc:
                     self.ui.error(six.text_type(exc))
             else:
-                query = http.query(dl_path, text=True)
+                query = http.query(dl_path, text=True, decode_body=decode_body)
         except SPMException as exc:
             self.ui.error(six.text_type(exc))
 

salt/utils/http.py

@@ -5,8 +5,6 @@ and the like, but also useful for basic HTTP testing.
 
 .. versionadded:: 2015.5.0
 """
-
-# Import python libs
 from __future__ import absolute_import, print_function, unicode_literals
 
 import cgi
@@ -20,15 +18,10 @@ import socket
 import ssl
 import zlib
 
-# Import salt libs
 import salt.config
-
-# pylint: disable=import-error,no-name-in-module
 import salt.ext.six.moves.http_client
 import salt.ext.six.moves.http_cookiejar
 import salt.ext.six.moves.urllib.request as urllib_request
-
-# Don't need a try/except block, since Salt depends on tornado
 import salt.ext.tornado.httputil
 import salt.ext.tornado.simple_httpclient
 import salt.loader
@@ -45,8 +38,6 @@ import salt.utils.xmlutil as xml
 import salt.utils.yaml
 import salt.version
 from salt._compat import ElementTree as ET
-
-# Import 3rd party libs
 from salt.ext import six
 from salt.ext.six.moves import StringIO
 from salt.ext.six.moves.urllib.error import URLError
@@ -58,8 +49,8 @@ from salt.template import compile_template
 from salt.utils.decorators.jinja import jinja_filter
 
 try:
-    from ssl import CertificateError  # pylint: disable=E0611
-    from ssl import match_hostname  # pylint: disable=E0611
+    from ssl import CertificateError
+    from ssl import match_hostname
 
     HAS_MATCHHOSTNAME = True
 except ImportError:
@@ -80,9 +71,6 @@ except ImportError:
     # pylint: enable=no-name-in-module
 
 
-# pylint: enable=import-error,no-name-in-module
-
-
 try:
     import salt.ext.tornado.curl_httpclient
 
@@ -190,6 +178,7 @@ def query(
     formdata=False,
     formdata_fieldname=None,
     formdata_filename=None,
+    decode_body=True,
     **kwargs
 ):
     """
@@ -412,7 +401,7 @@ def query(
         result_text = result.content
         result_cookies = result.cookies
         body = result.content
-        if not isinstance(body, six.text_type):
+        if not isinstance(body, six.text_type) and decode_body:
             body = body.decode(result.encoding or "utf-8")
         ret["body"] = body
     elif backend == "urllib2":
@@ -516,7 +505,7 @@ def query(
                 and not isinstance(result_text, six.text_type)
             ):
                 result_text = result_text.decode(res_params["charset"])
-        if six.PY3 and isinstance(result_text, bytes):
+        if six.PY3 and isinstance(result_text, bytes) and decode_body:
             result_text = result_text.decode("utf-8")
         ret["body"] = result_text
     else:
@@ -669,7 +658,7 @@ def query(
                 and not isinstance(result_text, six.text_type)
             ):
                 result_text = result_text.decode(res_params["charset"])
-        if six.PY3 and isinstance(result_text, bytes):
+        if six.PY3 and isinstance(result_text, bytes) and decode_body:
             result_text = result_text.decode("utf-8")
         ret["body"] = result_text
         if "Set-Cookie" in result_headers and cookies is not None:

tests/integration/spm/test_install.py

@@ -8,8 +8,11 @@ import os
 import shutil
 
 import pytest
+import salt.utils.files
+import salt.utils.path
+import salt.utils.yaml
 from tests.support.case import SPMCase
-from tests.support.helpers import destructiveTest, slowTest
+from tests.support.helpers import Webserver, destructiveTest, slowTest
 
 
 @destructiveTest
@@ -22,6 +25,40 @@ class SPMInstallTest(SPMCase):
     def setUp(self):
         self.config = self._spm_config()
         self._spm_build_files(self.config)
+        self.spm_build_dir = self.config["spm_build_dir"]
+        if "http" in self.id():
+            # only start the webserver when testing http
+            self.webserver = Webserver()
+            self.webserver.root = self.spm_build_dir
+            self.webserver.start()
+            self.repo_dir = self.config["spm_repos_config"] + ".d"
+            self.repo = os.path.join(self.repo_dir, "spm.repo")
+            url = {"my_repo": {"url": self.webserver.url("")[:-1]}}
+
+            if not os.path.exists(self.repo_dir):
+                os.makedirs(self.repo_dir)
+
+            with salt.utils.files.fopen(self.repo, "w") as fp:
+                salt.utils.yaml.safe_dump(url, fp)
+
+    def test_spm_install_http(self):
+        """
+        test spm install using http repo
+        """
+        build_spm = self.run_spm("build", self.config, self.formula_dir)
+        spm_file = os.path.join(self.spm_build_dir, "apache-201506-2.spm")
+
+        create_repo = self.run_spm("create_repo", self.config, self.spm_build_dir)
+
+        for root, dirs, files in salt.utils.path.os_walk(self.spm_build_dir):
+            for fp in files:
+                self.webserver.url(fp)
+
+        install = self.run_spm("install", self.config, "apache")
+
+        sls = os.path.join(self.config["formula_path"], "apache", "apache.sls")
+
+        self.assertTrue(os.path.exists(sls))
 
     @slowTest
     def test_spm_install_local_dir(self):
@@ -50,4 +87,6 @@ class SPMInstallTest(SPMCase):
         self.assertTrue(os.path.exists(sls))
 
     def tearDown(self):
+        if "http" in self.id():
+            self.webserver.stop()
         shutil.rmtree(self._tmp_spm)

tests/unit/utils/test_http.py

@@ -132,6 +132,24 @@ class HTTPTestCase(TestCase):
         self.assertTrue(isinstance(ret, dict))
         self.assertTrue(isinstance(ret.get("error", None), str))
 
+
+class HTTPPostTestCase(TestCase):
+    """
+    Unit TestCase for the salt.utils.http module when
+    using POST method
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        cls.post_webserver = Webserver(handler=MirrorPostHandler)
+        cls.post_webserver.start()
+        cls.post_web_root = cls.post_webserver.web_root
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.post_webserver.stop()
+        del cls.post_webserver
+
     def test_requests_multipart_formdata_post(self):
         """
         Test handling of a multipart/form-data POST using the requests backend
@@ -148,3 +166,46 @@ class HTTPTestCase(TestCase):
         body = ret.get("body", "")
         boundary = body[: body.find("\r")]
         self.assertEqual(body, match_this.format(boundary))
+
+
+class HTTPGetTestCase(TestCase):
+    """
+    Unit TestCase for the salt.utils.http module when
+    using Get method
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        cls.get_webserver = Webserver()
+        cls.get_webserver.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.get_webserver.stop()
+        del cls.get_webserver
+
+    def test_backends_decode_body_false(self):
+        """
+        test all backends when using
+        decode_body=False that it returns
+        bytes and does not try to decode
+        """
+        for backend in ["tornado", "requests", "urllib2"]:
+            ret = http.query(
+                self.get_webserver.url("custom.tar.gz"),
+                backend=backend,
+                decode_body=False,
+            )
+            body = ret.get("body", "")
+            assert isinstance(body, bytes)
+
+    def test_backends_decode_body_true(self):
+        """
+        test all backends when using
+        decode_body=True that it returns
+        string and decodes it.
+        """
+        for backend in ["tornado", "requests", "urllib2"]:
+            ret = http.query(self.get_webserver.url("core.sls"), backend=backend,)
+            body = ret.get("body", "")
+            assert isinstance(body, str)