sindoring/moonwell-client
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

обновление сборщика

Browse Source
This commit is contained in:
sindoring
2026-08-16 16:56:34 +04:00
parent 1a80ca8ad8
commit a984e705c3
17 changed files with 743 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
upload_to_s3.py
+167 -16
View File
@@ -11,9 +11,17 @@ import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
from patch_layout import PATCH_LAYOUT, normalized_path
S3_PREFIX = "World of Warcraft"
MANIFEST_S3_KEY = "manifest.json"
MANAGED_MANIFEST_S3_KEY = "moonwell-managed-manifest.json"
LEGACY_MANAGED_FILES = {
"utils/warcraftxlhost.exe",
"utils/warcraftxlhost.log",
"warcraftxl.before-texture-upload-fix.dll",
}
def compute_build_hash(files: list[dict]) -> str:
lines = [
@@ -25,15 +33,59 @@ def compute_build_hash(files: list[dict]) -> str:
def normalized_path_key(path: str) -> str:
"""Match client paths using Windows filesystem semantics."""
return path.replace("\\", "/").casefold()
return normalized_path(path)
def merge_manifests(base_manifest: dict, staging_manifest: dict) -> dict:
def missing_repository_patches(manifest: dict) -> list[str]:
paths = {
normalized_path_key(item["path"])
for item in manifest.get("files", [])
if isinstance(item, dict) and isinstance(item.get("path"), str)
}
return sorted(PATCH_LAYOUT.repository_patches - paths)
def external_graphics_patches(manifest: dict) -> list[str]:
return sorted(
item["path"]
for item in manifest.get("files", [])
if isinstance(item, dict)
and isinstance(item.get("path"), str)
and PATCH_LAYOUT.is_external_graphics_patch(item["path"])
)
def is_legacy_managed_path(path: str) -> bool:
key = normalized_path_key(path)
return (
key in LEGACY_MANAGED_FILES
or key in {"wow.exe", "d3d9.dll", "warcraftxl.dll", "utils/d3d9-native.dll"}
or key.startswith("extensions/")
or key.startswith("data/patch-wxl.mpq/")
)
def merge_manifests(
base_manifest: dict,
staging_manifest: dict,
previous_managed_paths: list[str] | None = None,
) -> dict:
previous_managed_keys = {
normalized_path_key(path) for path in (previous_managed_paths or [])
}
files_by_path = {
normalized_path_key(item["path"]): item
for item in base_manifest.get("files", [])
if not (
normalized_path_key(item["path"]) in previous_managed_keys
or is_legacy_managed_path(item["path"])
or PATCH_LAYOUT.is_repository_patch(item["path"])
)
or PATCH_LAYOUT.is_external_graphics_patch(item["path"])
}
for item in staging_manifest.get("files", []):
if PATCH_LAYOUT.is_external_graphics_patch(item["path"]):
continue
files_by_path[normalized_path_key(item["path"])] = item
files = sorted(files_by_path.values(), key=lambda item: item["path"].casefold())
@@ -43,6 +95,36 @@ def merge_manifests(base_manifest: dict, staging_manifest: dict) -> dict:
}
def find_stale_managed_paths(
base_manifest: dict,
staging_manifest: dict,
previous_managed_paths: list[str] | None = None,
) -> list[str]:
previous_managed_keys = {
normalized_path_key(path) for path in (previous_managed_paths or [])
}
staging_keys = {
normalized_path_key(item["path"])
for item in staging_manifest.get("files", [])
}
return sorted(
(
item["path"]
for item in base_manifest.get("files", [])
if (
(
normalized_path_key(item["path"]) in previous_managed_keys
or is_legacy_managed_path(item["path"])
or PATCH_LAYOUT.is_repository_patch(item["path"])
)
and not PATCH_LAYOUT.is_external_graphics_patch(item["path"])
)
and normalized_path_key(item["path"]) not in staging_keys
),
key=str.casefold,
)
def load_dotenv(env_path: Path) -> dict[str, str]:
values: dict[str, str] = {}
if not env_path.exists():
@@ -103,6 +185,24 @@ def main() -> int:
print(f"ERROR: unable to read local manifest.json: {error}", file=sys.stderr)
return 1
missing_repository = missing_repository_patches(staging_manifest)
if missing_repository:
print(
"ERROR: local manifest is missing repository patches: "
+ ", ".join(missing_repository),
file=sys.stderr,
)
return 1
unexpected_graphics = external_graphics_patches(staging_manifest)
if unexpected_graphics:
print(
"ERROR: managed manifest contains externally managed graphics patches: "
+ ", ".join(unexpected_graphics),
file=sys.stderr,
)
return 1
try:
response = s3.get_object(Bucket=bucket, Key=MANIFEST_S3_KEY)
production_manifest = json.loads(response["Body"].read())
@@ -116,7 +216,32 @@ def main() -> int:
print(f"ERROR: invalid production manifest: {error}", file=sys.stderr)
return 1
merged_manifest = merge_manifests(production_manifest, staging_manifest)
try:
response = s3.get_object(Bucket=bucket, Key=MANAGED_MANIFEST_S3_KEY)
previous_managed_manifest = json.loads(response["Body"].read())
previous_managed_paths = [
item["path"] for item in previous_managed_manifest.get("files", [])
]
except ClientError as error:
error_code = error.response.get("Error", {}).get("Code")
if error_code not in {"NoSuchKey", "404"}:
print(f"ERROR: unable to read managed manifest: {error}", file=sys.stderr)
return 1
previous_managed_paths = []
except (UnicodeDecodeError, json.JSONDecodeError, KeyError) as error:
print(f"ERROR: invalid managed manifest: {error}", file=sys.stderr)
return 1
merged_manifest = merge_manifests(
production_manifest,
staging_manifest,
previous_managed_paths,
)
stale_managed_paths = find_stale_managed_paths(
production_manifest,
staging_manifest,
previous_managed_paths,
)
print(
"Manifest merge: "
f"production={len(production_manifest.get('files', []))}, "
@@ -139,6 +264,9 @@ def main() -> int:
for local_path in files:
rel_path = local_path.relative_to(dist_dir).as_posix()
if PATCH_LAYOUT.is_external_graphics_patch(rel_path):
print(f"Skipping externally managed graphics patch: {rel_path}")
continue
s3_key = f"{S3_PREFIX}/{rel_path}"
size_mb = local_path.stat().st_size / (1024 * 1024)
print(f"Uploading {rel_path} ({size_mb:.1f} MB) -> s3://{bucket}/{s3_key}")
@@ -154,27 +282,50 @@ def main() -> int:
print("\nFinished with errors; production manifest was not changed.")
return 1
manifest_path.write_text(
json.dumps(merged_manifest, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
newline="\n",
json_content_type = "application/json"
no_cache = "no-cache, no-store, must-revalidate"
print(
f"Uploading managed package inventory -> "
f"s3://{bucket}/{MANAGED_MANIFEST_S3_KEY}"
)
try:
s3.put_object(
Bucket=bucket,
Key=MANAGED_MANIFEST_S3_KEY,
Body=(json.dumps(staging_manifest, ensure_ascii=False, indent=2) + "\n").encode("utf-8"),
ContentType=json_content_type,
CacheControl=no_cache,
)
print(" OK")
except Exception as e:
print(f" ERROR: {e}", file=sys.stderr)
return 1
print(f"Uploading manifest.json -> s3://{bucket}/{MANIFEST_S3_KEY}")
try:
s3.upload_file(
str(manifest_path),
bucket,
MANIFEST_S3_KEY,
ExtraArgs={
"ContentType": "application/json",
"CacheControl": "no-cache, no-store, must-revalidate",
},
s3.put_object(
Bucket=bucket,
Key=MANIFEST_S3_KEY,
Body=(json.dumps(merged_manifest, ensure_ascii=False, indent=2) + "\n").encode("utf-8"),
ContentType=json_content_type,
CacheControl=no_cache,
)
print(f" OK")
except Exception as e:
print(f" ERROR: {e}", file=sys.stderr)
errors = True
return 1
if stale_managed_paths:
print(f"Deleting {len(stale_managed_paths)} stale managed object(s)...")
for relative_path in stale_managed_paths:
s3_key = f"{S3_PREFIX}/{relative_path.replace(chr(92), '/')}"
try:
s3.delete_object(Bucket=bucket, Key=s3_key)
print(f" Deleted {s3_key}")
except Exception as e:
print(f" ERROR deleting {s3_key}: {e}", file=sys.stderr)
errors = True
if errors:
print("\nFinished with errors.")