feat(tooling): add mirror-airgap.sh (deterministic mirror + capture notice)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
213
tooling/mirror-airgap.sh
Executable file
213
tooling/mirror-airgap.sh
Executable file
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env bash
|
||||
# mirror-airgap.sh — populate the byte-exact upstream mirror for full client air-gap.
|
||||
#
|
||||
# Architecture (see plan/11-full-airgap-mirror.md): dnsmasq spoofs the real
|
||||
# upstream hostnames -> our host; Caddy serves byte-exact path mirrors over HTTPS
|
||||
# with `tls internal`. The launcher is UNMODIFIED — it requests the real Mojang
|
||||
# URLs and DNS resolves them to us. Mirror files live at:
|
||||
# mirror/upstream/<host>/<path> (== the real URL path, byte-for-byte)
|
||||
# `mirror/` is gitignored.
|
||||
#
|
||||
# ── What this script does DETERMINISTICALLY (no proxy needed) ────────────────
|
||||
# Everything reachable purely from the Mojang piston metadata graph for a given
|
||||
# MC version:
|
||||
# 1. version_manifest_v2.json (piston-meta.mojang.com)
|
||||
# 2. the per-version <id>.json (piston-meta.mojang.com)
|
||||
# 3. client.jar (piston-data.mojang.com) — sha1 verified
|
||||
# 4. every library artifact (libraries.minecraft.net / url-derived host) — sha1 verified
|
||||
# 5. the asset index (piston-meta / piston-data, real path) — sha1 verified
|
||||
# 6. every asset object (resources.download.minecraft.net/<2hex>/<hash>) — sha1 verified, idempotent
|
||||
# All downloads go to a .partial file, are sha1-checked, then atomically moved.
|
||||
# Any fetch failure or sha1 mismatch HALTS the script (set -euo pipefail).
|
||||
#
|
||||
# ── What this script CANNOT do here — the MANUAL PROXY-CAPTURE step ──────────
|
||||
# Two upstream surfaces are NOT enumerable from the piston graph and require the
|
||||
# online proxy-capture procedure documented in plan/11-full-airgap-mirror.md
|
||||
# ("Capture (the actual work)"):
|
||||
#
|
||||
# * meta.prismlauncher.org — Prism-format component metadata (the launcher's
|
||||
# own index of MC / LWJGL / NeoForge components).
|
||||
# Layout is launcher-specific (/v1/...), versioned,
|
||||
# and only knowable by observing what the launcher
|
||||
# actually requests.
|
||||
# * maven.neoforged.net — NeoForge installer + the NeoForge-processed
|
||||
# libraries (and possibly extra maven mirrors the
|
||||
# NeoForge install pulls). The exact artifact set
|
||||
# depends on the launcher/installer run.
|
||||
#
|
||||
# For those: launch the instance once online behind a logging forward/reverse
|
||||
# proxy (mitmproxy or a logging Caddy), collect every absolute URL hit, then
|
||||
# fetch each into mirror/upstream/<host>/<path> preserving the path exactly and
|
||||
# verifying sha1 where the source provides it. This script prints a MANUAL STEP
|
||||
# notice at the end as a reminder.
|
||||
#
|
||||
# Requires: curl, jq, sha1sum.
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.." # repo root
|
||||
|
||||
# --- env (optional) --------------------------------------------------------
|
||||
[ -f .env ] && { set -a; . ./.env; set +a; }
|
||||
|
||||
MC_VERSION="${MC_VERSION:-1.21.1}"
|
||||
NEOFORGE_VERSION="${NEOFORGE_VERSION:-21.1.209}" # informational; NeoForge libs come from capture
|
||||
|
||||
# --- required tools --------------------------------------------------------
|
||||
for tool in curl jq sha1sum; do
|
||||
command -v "$tool" >/dev/null 2>&1 || { echo "error: '$tool' is required but not found in PATH" >&2; exit 1; }
|
||||
done
|
||||
|
||||
MIRROR_ROOT="mirror/upstream"
|
||||
|
||||
MANIFEST_HOST="piston-meta.mojang.com"
|
||||
MANIFEST_PATH="mc/game/version_manifest_v2.json"
|
||||
MANIFEST_URL="https://${MANIFEST_HOST}/${MANIFEST_PATH}"
|
||||
RESOURCES_HOST="resources.download.minecraft.net"
|
||||
|
||||
echo "mirror-airgap: MC_VERSION=${MC_VERSION} NEOFORGE_VERSION=${NEOFORGE_VERSION} (NeoForge handled by capture)"
|
||||
echo "mirror root: ${MIRROR_ROOT}/"
|
||||
|
||||
# --- helpers ---------------------------------------------------------------
|
||||
sha1_of() { sha1sum "$1" | awk '{print $1}'; }
|
||||
|
||||
# host + path from an absolute https?:// URL (path has no leading slash).
|
||||
url_host() { printf '%s' "$1" | sed -E 's#^[a-zA-Z]+://([^/]+)/.*$#\1#'; }
|
||||
url_path() { printf '%s' "$1" | sed -E 's#^[a-zA-Z]+://[^/]+/##'; }
|
||||
|
||||
# Download $url into mirror/upstream/<host>/<path>, verifying sha1 ($want_sha1,
|
||||
# may be empty to skip verification). Idempotent: a present file whose sha1
|
||||
# already matches is skipped. Halts on any failure or mismatch.
|
||||
# Echoes the destination path on stdout (last line) for callers that need it.
|
||||
fetch_to_mirror() { # $1=url $2=want_sha1(optional)
|
||||
local url="$1" want="${2:-}"
|
||||
local host path dest tmp got
|
||||
host="$(url_host "$url")"
|
||||
path="$(url_path "$url")"
|
||||
[ -n "$host" ] || { echo "error: could not parse host from url: $url" >&2; exit 1; }
|
||||
[ -n "$path" ] || { echo "error: could not parse path from url: $url" >&2; exit 1; }
|
||||
dest="${MIRROR_ROOT}/${host}/${path}"
|
||||
|
||||
if [ -f "$dest" ]; then
|
||||
if [ -n "$want" ]; then
|
||||
got="$(sha1_of "$dest")"
|
||||
if [ "$got" = "$want" ]; then
|
||||
printf '%s\n' "$dest"
|
||||
return 0
|
||||
fi
|
||||
echo " re-fetch ${host}/${path} (local sha1 mismatch)" >&2
|
||||
else
|
||||
# no sha1 to check (e.g. manifest/version json) — re-fetch to stay fresh
|
||||
:
|
||||
fi
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$dest")"
|
||||
tmp="${dest}.partial"
|
||||
if ! curl -fSL --retry 3 -o "$tmp" "$url"; then
|
||||
rm -f "$tmp"
|
||||
echo "error: download failed: $url" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "$want" ]; then
|
||||
got="$(sha1_of "$tmp")"
|
||||
if [ "$got" != "$want" ]; then
|
||||
rm -f "$tmp"
|
||||
echo "error: sha1 mismatch for $url" >&2
|
||||
echo " expected: $want" >&2
|
||||
echo " got: $got" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
mv -f "$tmp" "$dest"
|
||||
printf '%s\n' "$dest"
|
||||
}
|
||||
|
||||
# --- 1. version manifest ---------------------------------------------------
|
||||
echo "[1/6] version manifest"
|
||||
manifest_dest="$(fetch_to_mirror "$MANIFEST_URL" "" | tail -n1)"
|
||||
|
||||
version_url="$(jq -r --arg v "$MC_VERSION" '.versions[] | select(.id == $v) | .url' "$manifest_dest")"
|
||||
[ -n "$version_url" ] && [ "$version_url" != "null" ] || {
|
||||
echo "error: MC version '$MC_VERSION' not found in version manifest" >&2; exit 1; }
|
||||
echo " version json: $version_url"
|
||||
|
||||
# --- 2. per-version json ---------------------------------------------------
|
||||
echo "[2/6] version json"
|
||||
version_dest="$(fetch_to_mirror "$version_url" "" | tail -n1)"
|
||||
|
||||
# --- 3. client.jar ---------------------------------------------------------
|
||||
echo "[3/6] client.jar"
|
||||
client_url="$(jq -r '.downloads.client.url' "$version_dest")"
|
||||
client_sha1="$(jq -r '.downloads.client.sha1' "$version_dest")"
|
||||
[ -n "$client_url" ] && [ "$client_url" != "null" ] || { echo "error: no .downloads.client.url in version json" >&2; exit 1; }
|
||||
fetch_to_mirror "$client_url" "$client_sha1" >/dev/null
|
||||
echo " ok client.jar (sha1 verified)"
|
||||
|
||||
# --- 4. libraries ----------------------------------------------------------
|
||||
echo "[4/6] libraries"
|
||||
lib_count="$(jq '[.libraries[] | select(.downloads.artifact != null)] | length' "$version_dest")"
|
||||
echo " $lib_count library artifact(s)"
|
||||
# Emit "url<TAB>sha1" per artifact; host+path derived from the url itself so
|
||||
# whatever maven host appears (libraries.minecraft.net or others) is mirrored.
|
||||
jq -r '.libraries[] | select(.downloads.artifact != null)
|
||||
| "\(.downloads.artifact.url)\t\(.downloads.artifact.sha1)"' "$version_dest" \
|
||||
| while IFS=$'\t' read -r lurl lsha1; do
|
||||
[ -n "$lurl" ] || continue
|
||||
fetch_to_mirror "$lurl" "$lsha1" >/dev/null
|
||||
done
|
||||
echo " ok libraries (sha1 verified)"
|
||||
|
||||
# --- 5. asset index --------------------------------------------------------
|
||||
echo "[5/6] asset index"
|
||||
ai_url="$(jq -r '.assetIndex.url' "$version_dest")"
|
||||
ai_sha1="$(jq -r '.assetIndex.sha1' "$version_dest")"
|
||||
ai_id="$(jq -r '.assetIndex.id' "$version_dest")"
|
||||
[ -n "$ai_url" ] && [ "$ai_url" != "null" ] || { echo "error: no .assetIndex.url in version json" >&2; exit 1; }
|
||||
echo " asset index id=$ai_id"
|
||||
ai_dest="$(fetch_to_mirror "$ai_url" "$ai_sha1" | tail -n1)"
|
||||
|
||||
# --- 6. asset objects ------------------------------------------------------
|
||||
echo "[6/6] asset objects (the bulk — this is the large/slow part)"
|
||||
obj_total="$(jq '.objects | length' "$ai_dest")"
|
||||
echo " $obj_total object(s) -> ${RESOURCES_HOST}/<2hex>/<hash>"
|
||||
n=0
|
||||
# Each object: hash -> served at resources.download.minecraft.net/<first2>/<hash>;
|
||||
# sha1 == the hash itself. Idempotent: fetch_to_mirror skips valid existing files.
|
||||
jq -r '.objects[].hash' "$ai_dest" \
|
||||
| while read -r hash; do
|
||||
[ -n "$hash" ] || continue
|
||||
n=$((n + 1))
|
||||
prefix="${hash:0:2}"
|
||||
obj_url="https://${RESOURCES_HOST}/${prefix}/${hash}"
|
||||
fetch_to_mirror "$obj_url" "$hash" >/dev/null
|
||||
done
|
||||
echo " ok asset objects (sha1 verified, idempotent)"
|
||||
|
||||
# --- manual step notice ----------------------------------------------------
|
||||
cat >&2 <<'NOTICE'
|
||||
|
||||
────────────────────────────────────────────────────────────────────────────
|
||||
MANUAL STEP REQUIRED — proxy-capture (see plan/11-full-airgap-mirror.md)
|
||||
────────────────────────────────────────────────────────────────────────────
|
||||
This script mirrored the deterministic Mojang piston graph only:
|
||||
piston-meta.mojang.com, piston-data.mojang.com, libraries.minecraft.net,
|
||||
resources.download.minecraft.net.
|
||||
|
||||
The following are NOT enumerable from the piston graph and were NOT fetched:
|
||||
* meta.prismlauncher.org — Prism-format component metadata (/v1/...)
|
||||
* maven.neoforged.net — NeoForge installer + processed libraries
|
||||
(plus any extra maven mirrors the install pulls)
|
||||
|
||||
To complete the air-gap mirror, run the capture procedure from
|
||||
plan/11-full-airgap-mirror.md ("Capture (the actual work)"):
|
||||
1. Launch the instance ONCE online behind a logging forward/reverse proxy
|
||||
(mitmproxy or a logging Caddy). Collect every absolute URL requested.
|
||||
2. Fetch each into mirror/upstream/<host>/<path>, preserving the path
|
||||
exactly; verify sha1 where the source json provides it.
|
||||
3. Offline re-test on a clean machine (internet cut, LAN DNS = dnsmasq,
|
||||
Caddy root CA trusted). Any 404 -> a missing URL -> add it to the fetch.
|
||||
────────────────────────────────────────────────────────────────────────────
|
||||
NOTICE
|
||||
|
||||
echo "deterministic mirror complete for MC ${MC_VERSION}."
|
||||
Reference in New Issue
Block a user