feat(tooling): add mirror-airgap.sh (deterministic mirror + capture notice)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
213
tooling/mirror-airgap.sh
Executable file
213
tooling/mirror-airgap.sh
Executable file
@@ -0,0 +1,213 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# mirror-airgap.sh — populate the byte-exact upstream mirror for full client air-gap.
|
||||||
|
#
|
||||||
|
# Architecture (see plan/11-full-airgap-mirror.md): dnsmasq spoofs the real
|
||||||
|
# upstream hostnames -> our host; Caddy serves byte-exact path mirrors over HTTPS
|
||||||
|
# with `tls internal`. The launcher is UNMODIFIED — it requests the real Mojang
|
||||||
|
# URLs and DNS resolves them to us. Mirror files live at:
|
||||||
|
# mirror/upstream/<host>/<path> (== the real URL path, byte-for-byte)
|
||||||
|
# `mirror/` is gitignored.
|
||||||
|
#
|
||||||
|
# ── What this script does DETERMINISTICALLY (no proxy needed) ────────────────
|
||||||
|
# Everything reachable purely from the Mojang piston metadata graph for a given
|
||||||
|
# MC version:
|
||||||
|
# 1. version_manifest_v2.json (piston-meta.mojang.com)
|
||||||
|
# 2. the per-version <id>.json (piston-meta.mojang.com)
|
||||||
|
# 3. client.jar (piston-data.mojang.com) — sha1 verified
|
||||||
|
# 4. every library artifact (libraries.minecraft.net / url-derived host) — sha1 verified
|
||||||
|
# 5. the asset index (piston-meta / piston-data, real path) — sha1 verified
|
||||||
|
# 6. every asset object (resources.download.minecraft.net/<2hex>/<hash>) — sha1 verified, idempotent
|
||||||
|
# All downloads go to a .partial file, are sha1-checked, then atomically moved.
|
||||||
|
# Any fetch failure or sha1 mismatch HALTS the script (set -euo pipefail).
|
||||||
|
#
|
||||||
|
# ── What this script CANNOT do here — the MANUAL PROXY-CAPTURE step ──────────
|
||||||
|
# Two upstream surfaces are NOT enumerable from the piston graph and require the
|
||||||
|
# online proxy-capture procedure documented in plan/11-full-airgap-mirror.md
|
||||||
|
# ("Capture (the actual work)"):
|
||||||
|
#
|
||||||
|
# * meta.prismlauncher.org — Prism-format component metadata (the launcher's
|
||||||
|
# own index of MC / LWJGL / NeoForge components).
|
||||||
|
# Layout is launcher-specific (/v1/...), versioned,
|
||||||
|
# and only knowable by observing what the launcher
|
||||||
|
# actually requests.
|
||||||
|
# * maven.neoforged.net — NeoForge installer + the NeoForge-processed
|
||||||
|
# libraries (and possibly extra maven mirrors the
|
||||||
|
# NeoForge install pulls). The exact artifact set
|
||||||
|
# depends on the launcher/installer run.
|
||||||
|
#
|
||||||
|
# For those: launch the instance once online behind a logging forward/reverse
|
||||||
|
# proxy (mitmproxy or a logging Caddy), collect every absolute URL hit, then
|
||||||
|
# fetch each into mirror/upstream/<host>/<path> preserving the path exactly and
|
||||||
|
# verifying sha1 where the source provides it. This script prints a MANUAL STEP
|
||||||
|
# notice at the end as a reminder.
|
||||||
|
#
|
||||||
|
# Requires: curl, jq, sha1sum.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
cd "$(dirname "$0")/.." # repo root
|
||||||
|
|
||||||
|
# --- env (optional) --------------------------------------------------------
|
||||||
|
[ -f .env ] && { set -a; . ./.env; set +a; }
|
||||||
|
|
||||||
|
MC_VERSION="${MC_VERSION:-1.21.1}"
|
||||||
|
NEOFORGE_VERSION="${NEOFORGE_VERSION:-21.1.209}" # informational; NeoForge libs come from capture
|
||||||
|
|
||||||
|
# --- required tools --------------------------------------------------------
|
||||||
|
for tool in curl jq sha1sum; do
|
||||||
|
command -v "$tool" >/dev/null 2>&1 || { echo "error: '$tool' is required but not found in PATH" >&2; exit 1; }
|
||||||
|
done
|
||||||
|
|
||||||
|
MIRROR_ROOT="mirror/upstream"
|
||||||
|
|
||||||
|
MANIFEST_HOST="piston-meta.mojang.com"
|
||||||
|
MANIFEST_PATH="mc/game/version_manifest_v2.json"
|
||||||
|
MANIFEST_URL="https://${MANIFEST_HOST}/${MANIFEST_PATH}"
|
||||||
|
RESOURCES_HOST="resources.download.minecraft.net"
|
||||||
|
|
||||||
|
echo "mirror-airgap: MC_VERSION=${MC_VERSION} NEOFORGE_VERSION=${NEOFORGE_VERSION} (NeoForge handled by capture)"
|
||||||
|
echo "mirror root: ${MIRROR_ROOT}/"
|
||||||
|
|
||||||
|
# --- helpers ---------------------------------------------------------------
|
||||||
|
sha1_of() { sha1sum "$1" | awk '{print $1}'; }
|
||||||
|
|
||||||
|
# host + path from an absolute https?:// URL (path has no leading slash).
|
||||||
|
url_host() { printf '%s' "$1" | sed -E 's#^[a-zA-Z]+://([^/]+)/.*$#\1#'; }
|
||||||
|
url_path() { printf '%s' "$1" | sed -E 's#^[a-zA-Z]+://[^/]+/##'; }
|
||||||
|
|
||||||
|
# Download $url into mirror/upstream/<host>/<path>, verifying sha1 ($want_sha1,
|
||||||
|
# may be empty to skip verification). Idempotent: a present file whose sha1
|
||||||
|
# already matches is skipped. Halts on any failure or mismatch.
|
||||||
|
# Echoes the destination path on stdout (last line) for callers that need it.
|
||||||
|
fetch_to_mirror() { # $1=url $2=want_sha1(optional)
|
||||||
|
local url="$1" want="${2:-}"
|
||||||
|
local host path dest tmp got
|
||||||
|
host="$(url_host "$url")"
|
||||||
|
path="$(url_path "$url")"
|
||||||
|
[ -n "$host" ] || { echo "error: could not parse host from url: $url" >&2; exit 1; }
|
||||||
|
[ -n "$path" ] || { echo "error: could not parse path from url: $url" >&2; exit 1; }
|
||||||
|
dest="${MIRROR_ROOT}/${host}/${path}"
|
||||||
|
|
||||||
|
if [ -f "$dest" ]; then
|
||||||
|
if [ -n "$want" ]; then
|
||||||
|
got="$(sha1_of "$dest")"
|
||||||
|
if [ "$got" = "$want" ]; then
|
||||||
|
printf '%s\n' "$dest"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo " re-fetch ${host}/${path} (local sha1 mismatch)" >&2
|
||||||
|
else
|
||||||
|
# no sha1 to check (e.g. manifest/version json) — re-fetch to stay fresh
|
||||||
|
:
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p "$(dirname "$dest")"
|
||||||
|
tmp="${dest}.partial"
|
||||||
|
if ! curl -fSL --retry 3 -o "$tmp" "$url"; then
|
||||||
|
rm -f "$tmp"
|
||||||
|
echo "error: download failed: $url" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "$want" ]; then
|
||||||
|
got="$(sha1_of "$tmp")"
|
||||||
|
if [ "$got" != "$want" ]; then
|
||||||
|
rm -f "$tmp"
|
||||||
|
echo "error: sha1 mismatch for $url" >&2
|
||||||
|
echo " expected: $want" >&2
|
||||||
|
echo " got: $got" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
mv -f "$tmp" "$dest"
|
||||||
|
printf '%s\n' "$dest"
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- 1. version manifest ---------------------------------------------------
|
||||||
|
echo "[1/6] version manifest"
|
||||||
|
manifest_dest="$(fetch_to_mirror "$MANIFEST_URL" "" | tail -n1)"
|
||||||
|
|
||||||
|
version_url="$(jq -r --arg v "$MC_VERSION" '.versions[] | select(.id == $v) | .url' "$manifest_dest")"
|
||||||
|
[ -n "$version_url" ] && [ "$version_url" != "null" ] || {
|
||||||
|
echo "error: MC version '$MC_VERSION' not found in version manifest" >&2; exit 1; }
|
||||||
|
echo " version json: $version_url"
|
||||||
|
|
||||||
|
# --- 2. per-version json ---------------------------------------------------
|
||||||
|
echo "[2/6] version json"
|
||||||
|
version_dest="$(fetch_to_mirror "$version_url" "" | tail -n1)"
|
||||||
|
|
||||||
|
# --- 3. client.jar ---------------------------------------------------------
|
||||||
|
echo "[3/6] client.jar"
|
||||||
|
client_url="$(jq -r '.downloads.client.url' "$version_dest")"
|
||||||
|
client_sha1="$(jq -r '.downloads.client.sha1' "$version_dest")"
|
||||||
|
[ -n "$client_url" ] && [ "$client_url" != "null" ] || { echo "error: no .downloads.client.url in version json" >&2; exit 1; }
|
||||||
|
fetch_to_mirror "$client_url" "$client_sha1" >/dev/null
|
||||||
|
echo " ok client.jar (sha1 verified)"
|
||||||
|
|
||||||
|
# --- 4. libraries ----------------------------------------------------------
|
||||||
|
echo "[4/6] libraries"
|
||||||
|
lib_count="$(jq '[.libraries[] | select(.downloads.artifact != null)] | length' "$version_dest")"
|
||||||
|
echo " $lib_count library artifact(s)"
|
||||||
|
# Emit "url<TAB>sha1" per artifact; host+path derived from the url itself so
|
||||||
|
# whatever maven host appears (libraries.minecraft.net or others) is mirrored.
|
||||||
|
jq -r '.libraries[] | select(.downloads.artifact != null)
|
||||||
|
| "\(.downloads.artifact.url)\t\(.downloads.artifact.sha1)"' "$version_dest" \
|
||||||
|
| while IFS=$'\t' read -r lurl lsha1; do
|
||||||
|
[ -n "$lurl" ] || continue
|
||||||
|
fetch_to_mirror "$lurl" "$lsha1" >/dev/null
|
||||||
|
done
|
||||||
|
echo " ok libraries (sha1 verified)"
|
||||||
|
|
||||||
|
# --- 5. asset index --------------------------------------------------------
|
||||||
|
echo "[5/6] asset index"
|
||||||
|
ai_url="$(jq -r '.assetIndex.url' "$version_dest")"
|
||||||
|
ai_sha1="$(jq -r '.assetIndex.sha1' "$version_dest")"
|
||||||
|
ai_id="$(jq -r '.assetIndex.id' "$version_dest")"
|
||||||
|
[ -n "$ai_url" ] && [ "$ai_url" != "null" ] || { echo "error: no .assetIndex.url in version json" >&2; exit 1; }
|
||||||
|
echo " asset index id=$ai_id"
|
||||||
|
ai_dest="$(fetch_to_mirror "$ai_url" "$ai_sha1" | tail -n1)"
|
||||||
|
|
||||||
|
# --- 6. asset objects ------------------------------------------------------
|
||||||
|
echo "[6/6] asset objects (the bulk — this is the large/slow part)"
|
||||||
|
obj_total="$(jq '.objects | length' "$ai_dest")"
|
||||||
|
echo " $obj_total object(s) -> ${RESOURCES_HOST}/<2hex>/<hash>"
|
||||||
|
n=0
|
||||||
|
# Each object: hash -> served at resources.download.minecraft.net/<first2>/<hash>;
|
||||||
|
# sha1 == the hash itself. Idempotent: fetch_to_mirror skips valid existing files.
|
||||||
|
jq -r '.objects[].hash' "$ai_dest" \
|
||||||
|
| while read -r hash; do
|
||||||
|
[ -n "$hash" ] || continue
|
||||||
|
n=$((n + 1))
|
||||||
|
prefix="${hash:0:2}"
|
||||||
|
obj_url="https://${RESOURCES_HOST}/${prefix}/${hash}"
|
||||||
|
fetch_to_mirror "$obj_url" "$hash" >/dev/null
|
||||||
|
done
|
||||||
|
echo " ok asset objects (sha1 verified, idempotent)"
|
||||||
|
|
||||||
|
# --- manual step notice ----------------------------------------------------
|
||||||
|
cat >&2 <<'NOTICE'
|
||||||
|
|
||||||
|
────────────────────────────────────────────────────────────────────────────
|
||||||
|
MANUAL STEP REQUIRED — proxy-capture (see plan/11-full-airgap-mirror.md)
|
||||||
|
────────────────────────────────────────────────────────────────────────────
|
||||||
|
This script mirrored the deterministic Mojang piston graph only:
|
||||||
|
piston-meta.mojang.com, piston-data.mojang.com, libraries.minecraft.net,
|
||||||
|
resources.download.minecraft.net.
|
||||||
|
|
||||||
|
The following are NOT enumerable from the piston graph and were NOT fetched:
|
||||||
|
* meta.prismlauncher.org — Prism-format component metadata (/v1/...)
|
||||||
|
* maven.neoforged.net — NeoForge installer + processed libraries
|
||||||
|
(plus any extra maven mirrors the install pulls)
|
||||||
|
|
||||||
|
To complete the air-gap mirror, run the capture procedure from
|
||||||
|
plan/11-full-airgap-mirror.md ("Capture (the actual work)"):
|
||||||
|
1. Launch the instance ONCE online behind a logging forward/reverse proxy
|
||||||
|
(mitmproxy or a logging Caddy). Collect every absolute URL requested.
|
||||||
|
2. Fetch each into mirror/upstream/<host>/<path>, preserving the path
|
||||||
|
exactly; verify sha1 where the source json provides it.
|
||||||
|
3. Offline re-test on a clean machine (internet cut, LAN DNS = dnsmasq,
|
||||||
|
Caddy root CA trusted). Any 404 -> a missing URL -> add it to the fetch.
|
||||||
|
────────────────────────────────────────────────────────────────────────────
|
||||||
|
NOTICE
|
||||||
|
|
||||||
|
echo "deterministic mirror complete for MC ${MC_VERSION}."
|
||||||
Reference in New Issue
Block a user