#!/usr/bin/env bash # mirror-airgap.sh — populate the byte-exact upstream mirror for full client air-gap. # # Architecture (see plan/11-full-airgap-mirror.md): dnsmasq spoofs the real # upstream hostnames -> our host; Caddy serves byte-exact path mirrors over HTTPS # with `tls internal`. The launcher is UNMODIFIED — it requests the real Mojang # URLs and DNS resolves them to us. Mirror files live at: # mirror/upstream// (== the real URL path, byte-for-byte) # `mirror/` is gitignored. # # ── What this script does DETERMINISTICALLY (no proxy needed) ──────────────── # Everything reachable purely from the Mojang piston metadata graph for a given # MC version: # 1. version_manifest_v2.json (piston-meta.mojang.com) # 2. the per-version .json (piston-meta.mojang.com) # 3. client.jar (piston-data.mojang.com) — sha1 verified # 4. every library artifact (libraries.minecraft.net / url-derived host) — sha1 verified # 5. the asset index (piston-meta / piston-data, real path) — sha1 verified # 6. every asset object (resources.download.minecraft.net/<2hex>/) — sha1 verified, idempotent # All downloads go to a .partial file, are sha1-checked, then atomically moved. # Any fetch failure or sha1 mismatch HALTS the script (set -euo pipefail). # # ── What this script CANNOT do here — the MANUAL PROXY-CAPTURE step ────────── # Two upstream surfaces are NOT enumerable from the piston graph and require the # online proxy-capture procedure documented in plan/11-full-airgap-mirror.md # ("Capture (the actual work)"): # # * meta.prismlauncher.org — Prism-format component metadata (the launcher's # own index of MC / LWJGL / NeoForge components). # Layout is launcher-specific (/v1/...), versioned, # and only knowable by observing what the launcher # actually requests. # * maven.neoforged.net — NeoForge installer + the NeoForge-processed # libraries (and possibly extra maven mirrors the # NeoForge install pulls). The exact artifact set # depends on the launcher/installer run. # # For those: launch the instance once online behind a logging forward/reverse # proxy (mitmproxy or a logging Caddy), collect every absolute URL hit, then # fetch each into mirror/upstream// preserving the path exactly and # verifying sha1 where the source provides it. This script prints a MANUAL STEP # notice at the end as a reminder. # # Requires: curl, jq, sha1sum. set -euo pipefail cd "$(dirname "$0")/.." # repo root # --- env (optional) -------------------------------------------------------- [ -f .env ] && { set -a; . ./.env; set +a; } MC_VERSION="${MC_VERSION:-1.21.1}" NEOFORGE_VERSION="${NEOFORGE_VERSION:-21.1.209}" # informational; NeoForge libs come from capture # --- required tools -------------------------------------------------------- for tool in curl jq sha1sum; do command -v "$tool" >/dev/null 2>&1 || { echo "error: '$tool' is required but not found in PATH" >&2; exit 1; } done MIRROR_ROOT="mirror/upstream" MANIFEST_HOST="piston-meta.mojang.com" MANIFEST_PATH="mc/game/version_manifest_v2.json" MANIFEST_URL="https://${MANIFEST_HOST}/${MANIFEST_PATH}" RESOURCES_HOST="resources.download.minecraft.net" echo "mirror-airgap: MC_VERSION=${MC_VERSION} NEOFORGE_VERSION=${NEOFORGE_VERSION} (NeoForge handled by capture)" echo "mirror root: ${MIRROR_ROOT}/" # --- helpers --------------------------------------------------------------- sha1_of() { sha1sum "$1" | awk '{print $1}'; } # host + path from an absolute https?:// URL (path has no leading slash). url_host() { printf '%s' "$1" | sed -E 's#^[a-zA-Z]+://([^/]+)/.*$#\1#'; } url_path() { printf '%s' "$1" | sed -E 's#^[a-zA-Z]+://[^/]+/##'; } # Download $url into mirror/upstream//, verifying sha1 ($want_sha1, # may be empty to skip verification). Idempotent: a present file whose sha1 # already matches is skipped. Halts on any failure or mismatch. # Echoes the destination path on stdout (last line) for callers that need it. fetch_to_mirror() { # $1=url $2=want_sha1(optional) local url="$1" want="${2:-}" local host path dest tmp got host="$(url_host "$url")" path="$(url_path "$url")" [ -n "$host" ] || { echo "error: could not parse host from url: $url" >&2; exit 1; } [ -n "$path" ] || { echo "error: could not parse path from url: $url" >&2; exit 1; } dest="${MIRROR_ROOT}/${host}/${path}" if [ -f "$dest" ]; then if [ -n "$want" ]; then got="$(sha1_of "$dest")" if [ "$got" = "$want" ]; then printf '%s\n' "$dest" return 0 fi echo " re-fetch ${host}/${path} (local sha1 mismatch)" >&2 else # no sha1 to check (e.g. manifest/version json) — re-fetch to stay fresh : fi fi mkdir -p "$(dirname "$dest")" tmp="${dest}.partial" if ! curl -fSL --retry 3 -o "$tmp" "$url"; then rm -f "$tmp" echo "error: download failed: $url" >&2 exit 1 fi if [ -n "$want" ]; then got="$(sha1_of "$tmp")" if [ "$got" != "$want" ]; then rm -f "$tmp" echo "error: sha1 mismatch for $url" >&2 echo " expected: $want" >&2 echo " got: $got" >&2 exit 1 fi fi mv -f "$tmp" "$dest" printf '%s\n' "$dest" } # --- 1. version manifest --------------------------------------------------- echo "[1/6] version manifest" manifest_dest="$(fetch_to_mirror "$MANIFEST_URL" "" | tail -n1)" version_url="$(jq -r --arg v "$MC_VERSION" '.versions[] | select(.id == $v) | .url' "$manifest_dest")" [ -n "$version_url" ] && [ "$version_url" != "null" ] || { echo "error: MC version '$MC_VERSION' not found in version manifest" >&2; exit 1; } echo " version json: $version_url" # --- 2. per-version json --------------------------------------------------- echo "[2/6] version json" version_dest="$(fetch_to_mirror "$version_url" "" | tail -n1)" # --- 3. client.jar --------------------------------------------------------- echo "[3/6] client.jar" client_url="$(jq -r '.downloads.client.url' "$version_dest")" client_sha1="$(jq -r '.downloads.client.sha1' "$version_dest")" [ -n "$client_url" ] && [ "$client_url" != "null" ] || { echo "error: no .downloads.client.url in version json" >&2; exit 1; } fetch_to_mirror "$client_url" "$client_sha1" >/dev/null echo " ok client.jar (sha1 verified)" # --- 4. libraries ---------------------------------------------------------- echo "[4/6] libraries" lib_count="$(jq '[.libraries[] | select(.downloads.artifact != null)] | length' "$version_dest")" echo " $lib_count library artifact(s)" # Emit "urlsha1" per artifact; host+path derived from the url itself so # whatever maven host appears (libraries.minecraft.net or others) is mirrored. jq -r '.libraries[] | select(.downloads.artifact != null) | "\(.downloads.artifact.url)\t\(.downloads.artifact.sha1)"' "$version_dest" \ | while IFS=$'\t' read -r lurl lsha1; do [ -n "$lurl" ] || continue fetch_to_mirror "$lurl" "$lsha1" >/dev/null done echo " ok libraries (sha1 verified)" # --- 5. asset index -------------------------------------------------------- echo "[5/6] asset index" ai_url="$(jq -r '.assetIndex.url' "$version_dest")" ai_sha1="$(jq -r '.assetIndex.sha1' "$version_dest")" ai_id="$(jq -r '.assetIndex.id' "$version_dest")" [ -n "$ai_url" ] && [ "$ai_url" != "null" ] || { echo "error: no .assetIndex.url in version json" >&2; exit 1; } echo " asset index id=$ai_id" ai_dest="$(fetch_to_mirror "$ai_url" "$ai_sha1" | tail -n1)" # --- 6. asset objects ------------------------------------------------------ echo "[6/6] asset objects (the bulk — this is the large/slow part)" obj_total="$(jq '.objects | length' "$ai_dest")" echo " $obj_total object(s) -> ${RESOURCES_HOST}/<2hex>/" n=0 # Each object: hash -> served at resources.download.minecraft.net//; # sha1 == the hash itself. Idempotent: fetch_to_mirror skips valid existing files. jq -r '.objects[].hash' "$ai_dest" \ | while read -r hash; do [ -n "$hash" ] || continue n=$((n + 1)) prefix="${hash:0:2}" obj_url="https://${RESOURCES_HOST}/${prefix}/${hash}" fetch_to_mirror "$obj_url" "$hash" >/dev/null done echo " ok asset objects (sha1 verified, idempotent)" # --- manual step notice ---------------------------------------------------- cat >&2 <<'NOTICE' ──────────────────────────────────────────────────────────────────────────── MANUAL STEP REQUIRED — proxy-capture (see plan/11-full-airgap-mirror.md) ──────────────────────────────────────────────────────────────────────────── This script mirrored the deterministic Mojang piston graph only: piston-meta.mojang.com, piston-data.mojang.com, libraries.minecraft.net, resources.download.minecraft.net. The following are NOT enumerable from the piston graph and were NOT fetched: * meta.prismlauncher.org — Prism-format component metadata (/v1/...) * maven.neoforged.net — NeoForge installer + processed libraries (plus any extra maven mirrors the install pulls) To complete the air-gap mirror, run the capture procedure from plan/11-full-airgap-mirror.md ("Capture (the actual work)"): 1. Launch the instance ONCE online behind a logging forward/reverse proxy (mitmproxy or a logging Caddy). Collect every absolute URL requested. 2. Fetch each into mirror/upstream//, preserving the path exactly; verify sha1 where the source json provides it. 3. Offline re-test on a clean machine (internet cut, LAN DNS = dnsmasq, Caddy root CA trusted). Any 404 -> a missing URL -> add it to the fetch. ──────────────────────────────────────────────────────────────────────────── NOTICE echo "deterministic mirror complete for MC ${MC_VERSION}."