diff --git a/tooling/mirror-airgap.sh b/tooling/mirror-airgap.sh new file mode 100755 index 0000000..85e10f0 --- /dev/null +++ b/tooling/mirror-airgap.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +# mirror-airgap.sh — populate the byte-exact upstream mirror for full client air-gap. +# +# Architecture (see plan/11-full-airgap-mirror.md): dnsmasq spoofs the real +# upstream hostnames -> our host; Caddy serves byte-exact path mirrors over HTTPS +# with `tls internal`. The launcher is UNMODIFIED — it requests the real Mojang +# URLs and DNS resolves them to us. Mirror files live at: +# mirror/upstream// (== the real URL path, byte-for-byte) +# `mirror/` is gitignored. +# +# ── What this script does DETERMINISTICALLY (no proxy needed) ──────────────── +# Everything reachable purely from the Mojang piston metadata graph for a given +# MC version: +# 1. version_manifest_v2.json (piston-meta.mojang.com) +# 2. the per-version .json (piston-meta.mojang.com) +# 3. client.jar (piston-data.mojang.com) — sha1 verified +# 4. every library artifact (libraries.minecraft.net / url-derived host) — sha1 verified +# 5. the asset index (piston-meta / piston-data, real path) — sha1 verified +# 6. every asset object (resources.download.minecraft.net/<2hex>/) — sha1 verified, idempotent +# All downloads go to a .partial file, are sha1-checked, then atomically moved. +# Any fetch failure or sha1 mismatch HALTS the script (set -euo pipefail). +# +# ── What this script CANNOT do here — the MANUAL PROXY-CAPTURE step ────────── +# Two upstream surfaces are NOT enumerable from the piston graph and require the +# online proxy-capture procedure documented in plan/11-full-airgap-mirror.md +# ("Capture (the actual work)"): +# +# * meta.prismlauncher.org — Prism-format component metadata (the launcher's +# own index of MC / LWJGL / NeoForge components). +# Layout is launcher-specific (/v1/...), versioned, +# and only knowable by observing what the launcher +# actually requests. +# * maven.neoforged.net — NeoForge installer + the NeoForge-processed +# libraries (and possibly extra maven mirrors the +# NeoForge install pulls). The exact artifact set +# depends on the launcher/installer run. +# +# For those: launch the instance once online behind a logging forward/reverse +# proxy (mitmproxy or a logging Caddy), collect every absolute URL hit, then +# fetch each into mirror/upstream// preserving the path exactly and +# verifying sha1 where the source provides it. This script prints a MANUAL STEP +# notice at the end as a reminder. +# +# Requires: curl, jq, sha1sum. +set -euo pipefail + +cd "$(dirname "$0")/.." # repo root + +# --- env (optional) -------------------------------------------------------- +[ -f .env ] && { set -a; . ./.env; set +a; } + +MC_VERSION="${MC_VERSION:-1.21.1}" +NEOFORGE_VERSION="${NEOFORGE_VERSION:-21.1.209}" # informational; NeoForge libs come from capture + +# --- required tools -------------------------------------------------------- +for tool in curl jq sha1sum; do + command -v "$tool" >/dev/null 2>&1 || { echo "error: '$tool' is required but not found in PATH" >&2; exit 1; } +done + +MIRROR_ROOT="mirror/upstream" + +MANIFEST_HOST="piston-meta.mojang.com" +MANIFEST_PATH="mc/game/version_manifest_v2.json" +MANIFEST_URL="https://${MANIFEST_HOST}/${MANIFEST_PATH}" +RESOURCES_HOST="resources.download.minecraft.net" + +echo "mirror-airgap: MC_VERSION=${MC_VERSION} NEOFORGE_VERSION=${NEOFORGE_VERSION} (NeoForge handled by capture)" +echo "mirror root: ${MIRROR_ROOT}/" + +# --- helpers --------------------------------------------------------------- +sha1_of() { sha1sum "$1" | awk '{print $1}'; } + +# host + path from an absolute https?:// URL (path has no leading slash). +url_host() { printf '%s' "$1" | sed -E 's#^[a-zA-Z]+://([^/]+)/.*$#\1#'; } +url_path() { printf '%s' "$1" | sed -E 's#^[a-zA-Z]+://[^/]+/##'; } + +# Download $url into mirror/upstream//, verifying sha1 ($want_sha1, +# may be empty to skip verification). Idempotent: a present file whose sha1 +# already matches is skipped. Halts on any failure or mismatch. +# Echoes the destination path on stdout (last line) for callers that need it. +fetch_to_mirror() { # $1=url $2=want_sha1(optional) + local url="$1" want="${2:-}" + local host path dest tmp got + host="$(url_host "$url")" + path="$(url_path "$url")" + [ -n "$host" ] || { echo "error: could not parse host from url: $url" >&2; exit 1; } + [ -n "$path" ] || { echo "error: could not parse path from url: $url" >&2; exit 1; } + dest="${MIRROR_ROOT}/${host}/${path}" + + if [ -f "$dest" ]; then + if [ -n "$want" ]; then + got="$(sha1_of "$dest")" + if [ "$got" = "$want" ]; then + printf '%s\n' "$dest" + return 0 + fi + echo " re-fetch ${host}/${path} (local sha1 mismatch)" >&2 + else + # no sha1 to check (e.g. manifest/version json) — re-fetch to stay fresh + : + fi + fi + + mkdir -p "$(dirname "$dest")" + tmp="${dest}.partial" + if ! curl -fSL --retry 3 -o "$tmp" "$url"; then + rm -f "$tmp" + echo "error: download failed: $url" >&2 + exit 1 + fi + + if [ -n "$want" ]; then + got="$(sha1_of "$tmp")" + if [ "$got" != "$want" ]; then + rm -f "$tmp" + echo "error: sha1 mismatch for $url" >&2 + echo " expected: $want" >&2 + echo " got: $got" >&2 + exit 1 + fi + fi + mv -f "$tmp" "$dest" + printf '%s\n' "$dest" +} + +# --- 1. version manifest --------------------------------------------------- +echo "[1/6] version manifest" +manifest_dest="$(fetch_to_mirror "$MANIFEST_URL" "" | tail -n1)" + +version_url="$(jq -r --arg v "$MC_VERSION" '.versions[] | select(.id == $v) | .url' "$manifest_dest")" +[ -n "$version_url" ] && [ "$version_url" != "null" ] || { + echo "error: MC version '$MC_VERSION' not found in version manifest" >&2; exit 1; } +echo " version json: $version_url" + +# --- 2. per-version json --------------------------------------------------- +echo "[2/6] version json" +version_dest="$(fetch_to_mirror "$version_url" "" | tail -n1)" + +# --- 3. client.jar --------------------------------------------------------- +echo "[3/6] client.jar" +client_url="$(jq -r '.downloads.client.url' "$version_dest")" +client_sha1="$(jq -r '.downloads.client.sha1' "$version_dest")" +[ -n "$client_url" ] && [ "$client_url" != "null" ] || { echo "error: no .downloads.client.url in version json" >&2; exit 1; } +fetch_to_mirror "$client_url" "$client_sha1" >/dev/null +echo " ok client.jar (sha1 verified)" + +# --- 4. libraries ---------------------------------------------------------- +echo "[4/6] libraries" +lib_count="$(jq '[.libraries[] | select(.downloads.artifact != null)] | length' "$version_dest")" +echo " $lib_count library artifact(s)" +# Emit "urlsha1" per artifact; host+path derived from the url itself so +# whatever maven host appears (libraries.minecraft.net or others) is mirrored. +jq -r '.libraries[] | select(.downloads.artifact != null) + | "\(.downloads.artifact.url)\t\(.downloads.artifact.sha1)"' "$version_dest" \ +| while IFS=$'\t' read -r lurl lsha1; do + [ -n "$lurl" ] || continue + fetch_to_mirror "$lurl" "$lsha1" >/dev/null + done +echo " ok libraries (sha1 verified)" + +# --- 5. asset index -------------------------------------------------------- +echo "[5/6] asset index" +ai_url="$(jq -r '.assetIndex.url' "$version_dest")" +ai_sha1="$(jq -r '.assetIndex.sha1' "$version_dest")" +ai_id="$(jq -r '.assetIndex.id' "$version_dest")" +[ -n "$ai_url" ] && [ "$ai_url" != "null" ] || { echo "error: no .assetIndex.url in version json" >&2; exit 1; } +echo " asset index id=$ai_id" +ai_dest="$(fetch_to_mirror "$ai_url" "$ai_sha1" | tail -n1)" + +# --- 6. asset objects ------------------------------------------------------ +echo "[6/6] asset objects (the bulk — this is the large/slow part)" +obj_total="$(jq '.objects | length' "$ai_dest")" +echo " $obj_total object(s) -> ${RESOURCES_HOST}/<2hex>/" +n=0 +# Each object: hash -> served at resources.download.minecraft.net//; +# sha1 == the hash itself. Idempotent: fetch_to_mirror skips valid existing files. +jq -r '.objects[].hash' "$ai_dest" \ +| while read -r hash; do + [ -n "$hash" ] || continue + n=$((n + 1)) + prefix="${hash:0:2}" + obj_url="https://${RESOURCES_HOST}/${prefix}/${hash}" + fetch_to_mirror "$obj_url" "$hash" >/dev/null + done +echo " ok asset objects (sha1 verified, idempotent)" + +# --- manual step notice ---------------------------------------------------- +cat >&2 <<'NOTICE' + +──────────────────────────────────────────────────────────────────────────── + MANUAL STEP REQUIRED — proxy-capture (see plan/11-full-airgap-mirror.md) +──────────────────────────────────────────────────────────────────────────── + This script mirrored the deterministic Mojang piston graph only: + piston-meta.mojang.com, piston-data.mojang.com, libraries.minecraft.net, + resources.download.minecraft.net. + + The following are NOT enumerable from the piston graph and were NOT fetched: + * meta.prismlauncher.org — Prism-format component metadata (/v1/...) + * maven.neoforged.net — NeoForge installer + processed libraries + (plus any extra maven mirrors the install pulls) + + To complete the air-gap mirror, run the capture procedure from + plan/11-full-airgap-mirror.md ("Capture (the actual work)"): + 1. Launch the instance ONCE online behind a logging forward/reverse proxy + (mitmproxy or a logging Caddy). Collect every absolute URL requested. + 2. Fetch each into mirror/upstream//, preserving the path + exactly; verify sha1 where the source json provides it. + 3. Offline re-test on a clean machine (internet cut, LAN DNS = dnsmasq, + Caddy root CA trusted). Any 404 -> a missing URL -> add it to the fetch. +──────────────────────────────────────────────────────────────────────────── +NOTICE + +echo "deterministic mirror complete for MC ${MC_VERSION}."