Create and use requirements/constraints.txt now that setuptools 69.0 broke builds again

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

noxfile.py

@@ -251,13 +251,15 @@ def _get_pip_requirements_file(session, crypto=None, requirements_type="ci"):
         session.error("Could not find a linux requirements file for {}".format(pydir))
 
 
-def _upgrade_pip_setuptools_and_wheel(session, upgrade=True, onedir=False):
+def _upgrade_pip_setuptools_and_wheel(session, upgrade=True):
     if SKIP_REQUIREMENTS_INSTALL:
         session.log(
             "Skipping Python Requirements because SKIP_REQUIREMENTS_INSTALL was found in the environ"
         )
         return False
 
+    env = os.environ.copy()
+    env["PIP_CONSTRAINT"] = str(REPO_ROOT / "requirements" / "constraints.txt")
     install_command = [
         "python",
         "-m",
@@ -267,20 +269,8 @@ def _upgrade_pip_setuptools_and_wheel(session, upgrade=True, onedir=False):
     ]
     if upgrade:
         install_command.append("-U")
-    if onedir:
-        requirements = [
-            "pip>=22.3.1,<23.0",
-            # https://github.com/pypa/setuptools/commit/137ab9d684075f772c322f455b0dd1f992ddcd8f
-            "setuptools>=65.6.3,<66",
-            "wheel",
-        ]
-    else:
-        requirements = [
-            "pip>=20.2.4,<21.2",
-            "setuptools!=50.*,!=51.*,!=52.*,<59",
-        ]
-    install_command.extend(requirements)
-    session_run_always(session, *install_command, silent=PIP_INSTALL_SILENT)
+    install_command.extend(["setuptools", "pip", "wheel"])
+    session_run_always(session, *install_command, silent=PIP_INSTALL_SILENT, env=env)
     return True
 
 
@@ -293,20 +283,23 @@ def _install_requirements(
     if onedir and IS_LINUX:
         session_run_always(session, "python3", "-m", "relenv", "toolchain", "fetch")
 
-    if not _upgrade_pip_setuptools_and_wheel(session, onedir=onedir):
+    if not _upgrade_pip_setuptools_and_wheel(session):
         return False
 
     # Install requirements
+    env = os.environ.copy()
+    env["PIP_CONSTRAINT"] = str(REPO_ROOT / "requirements" / "constraints.txt")
+
     requirements_file = _get_pip_requirements_file(
         session, requirements_type=requirements_type
     )
     install_command = ["--progress-bar=off", "-r", requirements_file]
-    session.install(*install_command, silent=PIP_INSTALL_SILENT)
+    session.install(*install_command, silent=PIP_INSTALL_SILENT, env=env)
 
     if extra_requirements:
         install_command = ["--progress-bar=off"]
         install_command += list(extra_requirements)
-        session.install(*install_command, silent=PIP_INSTALL_SILENT)
+        session.install(*install_command, silent=PIP_INSTALL_SILENT, env=env)
 
     if EXTRA_REQUIREMENTS_INSTALL:
         session.log(
@@ -318,13 +311,15 @@ def _install_requirements(
         # we're already using, we want to maintain the locked version
         install_command = ["--progress-bar=off", "--constraint", requirements_file]
         install_command += EXTRA_REQUIREMENTS_INSTALL.split()
-        session.install(*install_command, silent=PIP_INSTALL_SILENT)
+        session.install(*install_command, silent=PIP_INSTALL_SILENT, env=env)
 
     return True
 
 
 def _install_coverage_requirement(session):
     if SKIP_REQUIREMENTS_INSTALL is False:
+        env = os.environ.copy()
+        env["PIP_CONSTRAINT"] = str(REPO_ROOT / "requirements" / "constraints.txt")
         coverage_requirement = COVERAGE_REQUIREMENT
         if coverage_requirement is None:
             coverage_requirement = "coverage==7.3.1"
@@ -341,7 +336,10 @@ def _install_coverage_requirement(session):
                     # finish within 1 to 2 hours.
                     coverage_requirement = "coverage==5.5"
         session.install(
-            "--progress-bar=off", coverage_requirement, silent=PIP_INSTALL_SILENT
+            "--progress-bar=off",
+            coverage_requirement,
+            silent=PIP_INSTALL_SILENT,
+            env=env,
         )
 
 
@@ -1854,7 +1852,7 @@ def ci_test_onedir_pkgs(session):
         session_run_always(session, "python3", "-m", "relenv", "toolchain", "fetch")
 
     # Install requirements
-    if _upgrade_pip_setuptools_and_wheel(session, onedir=True):
+    if _upgrade_pip_setuptools_and_wheel(session):
         _install_requirements(session, "pyzmq")
     env = {
         "ONEDIR_TESTRUN": "1",

requirements/base.txt

@@ -1,3 +1,5 @@
+--constraint=constraints.txt
+
 Jinja2
 jmespath
 msgpack>=0.5,!=0.5.5

requirements/constraints.txt

@@ -0,0 +1,3 @@
+setuptools >=65.6.3,<66
+setuptools-scm < 8.0.0
+pip >=22.3.1,<23.0

tools/pkg/build.py

@@ -10,7 +10,6 @@ import os
 import pathlib
 import shutil
 import tarfile
-import tempfile
 import zipfile
 from typing import TYPE_CHECKING
 
@@ -98,18 +97,13 @@ def debian(
             os.environ[key] = value
             env_args.extend(["-e", key])
 
-    constraints = ["setuptools-scm<8"]
-    with tempfile.NamedTemporaryFile(
-        "w", prefix="reqs-constraints-", suffix=".txt", delete=False
-    ) as tfile:
-        with open(tfile.name, "w", encoding="utf-8") as wfh:
-            for req in constraints:
-                wfh.write(f"{req}\n")
-        env = os.environ.copy()
-        env["PIP_CONSTRAINT"] = str(tfile.name)
+    env = os.environ.copy()
+    env["PIP_CONSTRAINT"] = str(
+        tools.utils.REPO_ROOT / "requirements" / "constraints.txt"
+    )
 
-        ctx.run("ln", "-sf", "pkg/debian/", ".")
-        ctx.run("debuild", *env_args, "-uc", "-us", env=env)
+    ctx.run("ln", "-sf", "pkg/debian/", ".")
+    ctx.run("debuild", *env_args, "-uc", "-us", env=env)
 
     ctx.info("Done")
 
@@ -174,20 +168,14 @@ def rpm(
         for key, value in new_env.items():
             os.environ[key] = value
 
-    constraints = ["setuptools-scm<8"]
-    with tempfile.NamedTemporaryFile(
-        "w", prefix="reqs-constraints-", suffix=".txt", delete=False
-    ) as tfile:
-        with open(tfile.name, "w", encoding="utf-8") as wfh:
-            for req in constraints:
-                wfh.write(f"{req}\n")
-        env = os.environ.copy()
-        env["PIP_CONSTRAINT"] = str(tfile.name)
-
-        spec_file = checkout / "pkg" / "rpm" / "salt.spec"
-        ctx.run(
-            "rpmbuild", "-bb", f"--define=_salt_src {checkout}", str(spec_file), env=env
-        )
+    env = os.environ.copy()
+    env["PIP_CONSTRAINT"] = str(
+        tools.utils.REPO_ROOT / "requirements" / "constraints.txt"
+    )
+    spec_file = checkout / "pkg" / "rpm" / "salt.spec"
+    ctx.run(
+        "rpmbuild", "-bb", f"--define=_salt_src {checkout}", str(spec_file), env=env
+    )
 
     ctx.info("Done")
 
@@ -575,51 +563,31 @@ def onedir_dependencies(
     )
     _check_pkg_build_files_exist(ctx, requirements_file=requirements_file)
 
-    constraints = ["setuptools-scm<8"]
-    with tempfile.NamedTemporaryFile(
-        "w", prefix="reqs-constraints-", suffix=".txt", delete=False
-    ) as tfile:
-        with open(tfile.name, "w", encoding="utf-8") as wfh:
-            for req in constraints:
-                wfh.write(f"{req}\n")
-        env["PIP_CONSTRAINT"] = str(tfile.name)
-        ctx.run(
-            str(python_bin),
-            "-m",
-            "pip",
-            "install",
-            "-U",
-            "wheel",
-            env=env,
-        )
-        ctx.run(
-            str(python_bin),
-            "-m",
-            "pip",
-            "install",
-            "-U",
-            "pip>=22.3.1,<23.0",
-            env=env,
-        )
-        ctx.run(
-            str(python_bin),
-            "-m",
-            "pip",
-            "install",
-            "-U",
-            "setuptools>=65.6.3,<66",
-            env=env,
-        )
-        ctx.run(
-            str(python_bin),
-            "-m",
-            "pip",
-            "install",
-            *install_args,
-            "-r",
-            str(requirements_file),
-            env=env,
-        )
+    env = os.environ.copy()
+    env["PIP_CONSTRAINT"] = str(
+        tools.utils.REPO_ROOT / "requirements" / "constraints.txt"
+    )
+    ctx.run(
+        str(python_bin),
+        "-m",
+        "pip",
+        "install",
+        "-U",
+        "setuptools",
+        "pip",
+        "wheel",
+        env=env,
+    )
+    ctx.run(
+        str(python_bin),
+        "-m",
+        "pip",
+        "install",
+        *install_args,
+        "-r",
+        str(requirements_file),
+        env=env,
+    )
 
 
 @build.command(