mirror of
https://github.com/saltstack/salt.git
synced 2025-04-17 10:10:20 +00:00
tools.utils is now a package
Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>
This commit is contained in:
Pedro Algarvio
Megan Wilhite
parent
d7d5bb30be
commit
4f30001ca5
4 changed files with 143 additions and 136 deletions
|
|
@ -19,12 +19,11 @@ from ptscripts import Context, command_group
|
|||
|
||||
import tools.pkg
|
||||
import tools.utils
|
||||
from tools.utils import (
|
||||
Version,
|
||||
from tools.utils import Version, parse_versions
|
||||
from tools.utils.repo import (
|
||||
create_full_repo_path,
|
||||
create_top_level_repo_path,
|
||||
get_repo_json_file_contents,
|
||||
parse_versions,
|
||||
)
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -20,13 +20,9 @@ from ptscripts import Context, command_group
|
|||
|
||||
import tools.pkg
|
||||
import tools.utils
|
||||
from tools.utils import (
|
||||
Version,
|
||||
create_full_repo_path,
|
||||
get_repo_json_file_contents,
|
||||
get_salt_releases,
|
||||
parse_versions,
|
||||
)
|
||||
import tools.utils.repo
|
||||
from tools.utils import Version, get_salt_releases, parse_versions
|
||||
from tools.utils.repo import create_full_repo_path, get_repo_json_file_contents
|
||||
|
||||
try:
|
||||
import boto3
|
||||
|
|
@ -336,7 +332,7 @@ def release(ctx: Context, salt_version: str):
|
|||
Bucket=bucket_name,
|
||||
Key=path,
|
||||
Fileobj=wfh,
|
||||
Callback=tools.utils.UpdateProgress(progress, task),
|
||||
Callback=tools.utils.repo.UpdateProgress(progress, task),
|
||||
)
|
||||
updated_contents = re.sub(
|
||||
r"^(baseurl|gpgkey)=https://([^/]+)/(.*)$",
|
||||
|
|
@ -366,7 +362,7 @@ def release(ctx: Context, salt_version: str):
|
|||
str(upload_path),
|
||||
tools.utils.RELEASE_BUCKET_NAME,
|
||||
str(relpath),
|
||||
Callback=tools.utils.UpdateProgress(progress, task),
|
||||
Callback=tools.utils.repo.UpdateProgress(progress, task),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -438,7 +434,7 @@ def github(
|
|||
Bucket=tools.utils.STAGING_BUCKET_NAME,
|
||||
Key=str(entry_path),
|
||||
Fileobj=wfh,
|
||||
Callback=tools.utils.UpdateProgress(progress, task),
|
||||
Callback=tools.utils.repo.UpdateProgress(progress, task),
|
||||
)
|
||||
|
||||
for artifact in artifacts_path.iterdir():
|
||||
|
|
@ -618,7 +614,7 @@ def _publish_repo(
|
|||
str(upload_path),
|
||||
bucket_name,
|
||||
str(relpath),
|
||||
Callback=tools.utils.UpdateProgress(progress, task),
|
||||
Callback=tools.utils.repo.UpdateProgress(progress, task),
|
||||
ExtraArgs={
|
||||
"Metadata": {
|
||||
"x-amz-meta-salt-release-version": salt_version,
|
||||
|
|
|
|||
|
|
@ -25,19 +25,7 @@ from rich.progress import (
|
|||
TransferSpeedColumn,
|
||||
)
|
||||
|
||||
try:
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
except ImportError:
|
||||
print(
|
||||
"\nPlease run 'python -m pip install -r "
|
||||
"requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info),
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
raise
|
||||
|
||||
REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||||
REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent
|
||||
GPG_KEY_FILENAME = "SALT-PROJECT-GPG-PUBKEY-2023"
|
||||
SPB_ENVIRONMENT = os.environ.get("SPB_ENVIRONMENT") or "test"
|
||||
STAGING_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-staging"
|
||||
|
|
@ -45,15 +33,6 @@ RELEASE_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-release"
|
|||
BACKUP_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-backup"
|
||||
|
||||
|
||||
class UpdateProgress:
|
||||
def __init__(self, progress, task):
|
||||
self.progress = progress
|
||||
self.task = task
|
||||
|
||||
def __call__(self, chunk_size):
|
||||
self.progress.update(self.task, advance=chunk_size)
|
||||
|
||||
|
||||
def create_progress_bar(file_progress: bool = False, **kwargs):
|
||||
if file_progress:
|
||||
return Progress(
|
||||
|
|
@ -203,106 +182,6 @@ def parse_versions(*versions: str) -> list[Version]:
|
|||
return _versions
|
||||
|
||||
|
||||
def get_repo_json_file_contents(
|
||||
ctx: Context,
|
||||
bucket_name: str,
|
||||
repo_path: pathlib.Path,
|
||||
repo_json_path: pathlib.Path,
|
||||
) -> dict[str, Any]:
|
||||
s3 = boto3.client("s3")
|
||||
repo_json: dict[str, Any] = {}
|
||||
try:
|
||||
ret = s3.head_object(
|
||||
Bucket=bucket_name, Key=str(repo_json_path.relative_to(repo_path))
|
||||
)
|
||||
ctx.info(
|
||||
f"Downloading existing '{repo_json_path.relative_to(repo_path)}' file "
|
||||
f"from bucket {bucket_name}"
|
||||
)
|
||||
size = ret["ContentLength"]
|
||||
with repo_json_path.open("wb") as wfh:
|
||||
with create_progress_bar(file_progress=True) as progress:
|
||||
task = progress.add_task(description="Downloading...", total=size)
|
||||
s3.download_fileobj(
|
||||
Bucket=bucket_name,
|
||||
Key=str(repo_json_path.relative_to(repo_path)),
|
||||
Fileobj=wfh,
|
||||
Callback=UpdateProgress(progress, task),
|
||||
)
|
||||
with repo_json_path.open() as rfh:
|
||||
repo_json = json.load(rfh)
|
||||
except ClientError as exc:
|
||||
if "Error" not in exc.response:
|
||||
raise
|
||||
if exc.response["Error"]["Code"] != "404":
|
||||
raise
|
||||
ctx.info(f"Could not find {repo_json_path} in bucket {bucket_name}")
|
||||
if repo_json:
|
||||
ctx.print(repo_json, soft_wrap=True)
|
||||
return repo_json
|
||||
|
||||
|
||||
def create_top_level_repo_path(
|
||||
ctx: Context,
|
||||
repo_path: pathlib.Path,
|
||||
salt_version: str,
|
||||
distro: str,
|
||||
distro_version: str | None = None, # pylint: disable=bad-whitespace
|
||||
distro_arch: str | None = None, # pylint: disable=bad-whitespace
|
||||
nightly_build_from: str | None = None, # pylint: disable=bad-whitespace
|
||||
):
|
||||
create_repo_path = repo_path
|
||||
if nightly_build_from:
|
||||
create_repo_path = (
|
||||
create_repo_path
|
||||
/ "salt-dev"
|
||||
/ nightly_build_from
|
||||
/ datetime.utcnow().strftime("%Y-%m-%d")
|
||||
)
|
||||
create_repo_path.mkdir(exist_ok=True, parents=True)
|
||||
with ctx.chdir(create_repo_path.parent):
|
||||
latest_nightly_symlink = pathlib.Path("latest")
|
||||
if not latest_nightly_symlink.exists():
|
||||
ctx.info(
|
||||
f"Creating 'latest' symlink to '{create_repo_path.relative_to(repo_path)}' ..."
|
||||
)
|
||||
latest_nightly_symlink.symlink_to(
|
||||
create_repo_path.name, target_is_directory=True
|
||||
)
|
||||
elif "rc" in salt_version:
|
||||
create_repo_path = create_repo_path / "salt_rc"
|
||||
create_repo_path = create_repo_path / "salt" / "py3" / distro
|
||||
if distro_version:
|
||||
create_repo_path = create_repo_path / distro_version
|
||||
if distro_arch:
|
||||
create_repo_path = create_repo_path / distro_arch
|
||||
create_repo_path.mkdir(exist_ok=True, parents=True)
|
||||
return create_repo_path
|
||||
|
||||
|
||||
def create_full_repo_path(
|
||||
ctx: Context,
|
||||
repo_path: pathlib.Path,
|
||||
salt_version: str,
|
||||
distro: str,
|
||||
distro_version: str | None = None, # pylint: disable=bad-whitespace
|
||||
distro_arch: str | None = None, # pylint: disable=bad-whitespace
|
||||
nightly_build_from: str | None = None, # pylint: disable=bad-whitespace
|
||||
):
|
||||
create_repo_path = create_top_level_repo_path(
|
||||
ctx,
|
||||
repo_path,
|
||||
salt_version,
|
||||
distro,
|
||||
distro_version,
|
||||
distro_arch,
|
||||
nightly_build_from=nightly_build_from,
|
||||
)
|
||||
create_repo_path = create_repo_path / "minor" / salt_version
|
||||
create_repo_path.mkdir(exist_ok=True, parents=True)
|
||||
return create_repo_path
|
||||
|
||||
|
||||
def get_file_checksum(fpath: pathlib.Path, hash_name: str) -> str:
|
||||
with fpath.open("rb") as rfh:
|
||||
try:
|
||||
133
tools/utils/repo.py
Normal file
133
tools/utils/repo.py
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated,bad-whitespace
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from ptscripts import Context
|
||||
|
||||
import tools.utils
|
||||
|
||||
try:
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
except ImportError:
|
||||
print(
|
||||
"\nPlease run 'python -m pip install -r "
|
||||
"requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info),
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
class UpdateProgress:
|
||||
def __init__(self, progress, task):
|
||||
self.progress = progress
|
||||
self.task = task
|
||||
|
||||
def __call__(self, chunk_size):
|
||||
self.progress.update(self.task, advance=chunk_size)
|
||||
|
||||
|
||||
def get_repo_json_file_contents(
|
||||
ctx: Context,
|
||||
bucket_name: str,
|
||||
repo_path: pathlib.Path,
|
||||
repo_json_path: pathlib.Path,
|
||||
) -> dict[str, Any]:
|
||||
s3 = boto3.client("s3")
|
||||
repo_json: dict[str, Any] = {}
|
||||
try:
|
||||
ret = s3.head_object(
|
||||
Bucket=bucket_name, Key=str(repo_json_path.relative_to(repo_path))
|
||||
)
|
||||
ctx.info(
|
||||
f"Downloading existing '{repo_json_path.relative_to(repo_path)}' file "
|
||||
f"from bucket {bucket_name}"
|
||||
)
|
||||
size = ret["ContentLength"]
|
||||
with repo_json_path.open("wb") as wfh:
|
||||
with tools.utils.create_progress_bar(file_progress=True) as progress:
|
||||
task = progress.add_task(description="Downloading...", total=size)
|
||||
s3.download_fileobj(
|
||||
Bucket=bucket_name,
|
||||
Key=str(repo_json_path.relative_to(repo_path)),
|
||||
Fileobj=wfh,
|
||||
Callback=UpdateProgress(progress, task),
|
||||
)
|
||||
with repo_json_path.open() as rfh:
|
||||
repo_json = json.load(rfh)
|
||||
except ClientError as exc:
|
||||
if "Error" not in exc.response:
|
||||
raise
|
||||
if exc.response["Error"]["Code"] != "404":
|
||||
raise
|
||||
ctx.info(f"Could not find {repo_json_path} in bucket {bucket_name}")
|
||||
if repo_json:
|
||||
ctx.print(repo_json, soft_wrap=True)
|
||||
return repo_json
|
||||
|
||||
|
||||
def create_top_level_repo_path(
|
||||
ctx: Context,
|
||||
repo_path: pathlib.Path,
|
||||
salt_version: str,
|
||||
distro: str,
|
||||
distro_version: str | None = None, # pylint: disable=bad-whitespace
|
||||
distro_arch: str | None = None, # pylint: disable=bad-whitespace
|
||||
nightly_build_from: str | None = None, # pylint: disable=bad-whitespace
|
||||
):
|
||||
create_repo_path = repo_path
|
||||
if nightly_build_from:
|
||||
create_repo_path = (
|
||||
create_repo_path
|
||||
/ "salt-dev"
|
||||
/ nightly_build_from
|
||||
/ datetime.utcnow().strftime("%Y-%m-%d")
|
||||
)
|
||||
create_repo_path.mkdir(exist_ok=True, parents=True)
|
||||
with ctx.chdir(create_repo_path.parent):
|
||||
latest_nightly_symlink = pathlib.Path("latest")
|
||||
if not latest_nightly_symlink.exists():
|
||||
ctx.info(
|
||||
f"Creating 'latest' symlink to '{create_repo_path.relative_to(repo_path)}' ..."
|
||||
)
|
||||
latest_nightly_symlink.symlink_to(
|
||||
create_repo_path.name, target_is_directory=True
|
||||
)
|
||||
elif "rc" in salt_version:
|
||||
create_repo_path = create_repo_path / "salt_rc"
|
||||
create_repo_path = create_repo_path / "salt" / "py3" / distro
|
||||
if distro_version:
|
||||
create_repo_path = create_repo_path / distro_version
|
||||
if distro_arch:
|
||||
create_repo_path = create_repo_path / distro_arch
|
||||
create_repo_path.mkdir(exist_ok=True, parents=True)
|
||||
return create_repo_path
|
||||
|
||||
|
||||
def create_full_repo_path(
|
||||
ctx: Context,
|
||||
repo_path: pathlib.Path,
|
||||
salt_version: str,
|
||||
distro: str,
|
||||
distro_version: str | None = None, # pylint: disable=bad-whitespace
|
||||
distro_arch: str | None = None, # pylint: disable=bad-whitespace
|
||||
nightly_build_from: str | None = None, # pylint: disable=bad-whitespace
|
||||
):
|
||||
create_repo_path = create_top_level_repo_path(
|
||||
ctx,
|
||||
repo_path,
|
||||
salt_version,
|
||||
distro,
|
||||
distro_version,
|
||||
distro_arch,
|
||||
nightly_build_from=nightly_build_from,
|
||||
)
|
||||
create_repo_path = create_repo_path / "minor" / salt_version
|
||||
create_repo_path.mkdir(exist_ok=True, parents=True)
|
||||
return create_repo_path
|
||||
Loading…
Add table
Reference in a new issue