Add output caching for Forge 1.13+.

CHANGES
 - Forge installer is no longer stored to the work directory. It is now stored in a cache folder corresponding to the artifact version.
 - Installer output is now cached by default
 - Added options to generate distro.
   - --discardOutput Delete cached output after it is no longer required. May be useful if disk space is limited.
   - --invalidateCache Invalidate and delete existing caches as they are encountered. Requires fresh cache generation.
   - Both options are false by default.
 - To invalide a single version, manually delete the folder.
 - Old functionality is essentially g distro --discardOutput --invalidateCache.

README.md

@@ -138,6 +138,14 @@ Options:
   * OPTIONAL (default: false)
   * This is useful to easily test the new distribution.json in dev mode on Helios.
   * Tip: Set name to `dev_distribution` when using this option.
+* `--discardOutput` Delete cached output after it is no longer required. May be useful if disk space is limited.
+  * OPTIONAL (default: false)
+* `--invalidateCache` Invalidate and delete existing caches as they are encountered. Requires fresh cache generation.
+  * OPTIONAL (default: false)
+
+#### Notes
+
+As of Forge 1.13, the installer must be run to generate required files. The installer output is cached by default. This is done to speed up subsequent builds and allow Nebula to be run as a CI job. Options are provided to discard installer output (no caching) and invalidate caches (delete cached output and require fresh generation). To invalidate only a single version cache, manually delete the cached folder.
 
 >
 > Example Usage

src/index.ts

@@ -51,6 +51,26 @@ function installLocalOption(yargs: yargs.Argv): yargs.Argv {
     })
 }
 
+function discardOutputOption(yargs: yargs.Argv): yargs.Argv {
+    return yargs.option('discardOutput', {
+        describe: 'Delete cached output after it is no longer required. May be useful if disk space is limited.',
+        type: 'boolean',
+        demandOption: false,
+        global: false,
+        default: false
+    })
+}
+
+function invalidateCacheOption(yargs: yargs.Argv): yargs.Argv {
+    return yargs.option('invalidateCache', {
+        describe: 'Invalidate and delete existing caches as they are encountered. Requires fresh cache generation.',
+        type: 'boolean',
+        demandOption: false,
+        global: false,
+        default: false
+    })
+}
+
 // function rootOption(yargs: yargs.Argv) {
 //     return yargs.option('root', {
 //         describe: 'File structure root.',
@@ -111,7 +131,7 @@ const initRootCommand: yargs.CommandModule = {
         logger.debug('Invoked init root.')
         try {
             await generateSchemas(argv.root as string)
-            await new DistributionStructure(argv.root as string, '').init()
+            await new DistributionStructure(argv.root as string, '', false, false).init()
             logger.info(`Successfully created new root at ${argv.root}`)
         } catch (error) {
             logger.error(`Failed to init new root at ${argv.root}`, error)
@@ -179,7 +199,7 @@ const generateServerCommand: yargs.CommandModule = {
             }
         }
 
-        const serverStruct = new ServerStructure(argv.root as string, getBaseURL())
+        const serverStruct = new ServerStructure(argv.root as string, getBaseURL(), false, false)
         serverStruct.createServer(
             argv.id as string,
             minecraftVersion,
@@ -197,6 +217,8 @@ const generateDistroCommand: yargs.CommandModule = {
     describe: 'Generate a distribution index from the root file structure.',
     builder: (yargs) => {
         yargs = installLocalOption(yargs)
+        yargs = discardOutputOption(yargs)
+        yargs = invalidateCacheOption(yargs)
         yargs = namePositional(yargs)
         return yargs
     },
@@ -209,9 +231,13 @@ const generateDistroCommand: yargs.CommandModule = {
         logger.debug(`Root set to ${argv.root}`)
         logger.debug(`Base Url set to ${argv.baseUrl}`)
         logger.debug(`Install option set to ${argv.installLocal}`)
+        logger.debug(`Discard Output option set to ${argv.discardOutput}`)
+        logger.debug(`Invalidate Cache option set to ${argv.invalidateCache}`)
         logger.debug(`Invoked generate distro name ${finalName}.`)
 
         const doLocalInstall = argv.installLocal as boolean
+        const discardOutput = argv.discardOutput as boolean ?? false
+        const invalidateCache = argv.invalidateCache as boolean ?? false
         const heliosDataFolder = getHeliosDataFolder()
         if(doLocalInstall && heliosDataFolder == null) {
             logger.error('You MUST specify HELIOS_DATA_FOLDER in your .env when using the --installLocal option.')
@@ -219,7 +245,7 @@ const generateDistroCommand: yargs.CommandModule = {
         }
 
         try {
-            const distributionStruct = new DistributionStructure(argv.root as string, argv.baseUrl as string)
+            const distributionStruct = new DistributionStructure(argv.root as string, argv.baseUrl as string, discardOutput, invalidateCache)
             const distro = await distributionStruct.getSpecModel()
             const distroOut = JSON.stringify(distro, null, 2)
             const distroPath = resolvePath(argv.root as string, finalName)
@@ -333,7 +359,7 @@ const testCommand: yargs.CommandModule = {
         logger.info(process.cwd())
         const mcVer = new MinecraftVersion(argv.mcVer as string)
         const resolver = VersionSegmentedRegistry.getForgeResolver(mcVer,
-            argv.forgeVer as string, getRoot(), '', getBaseURL())
+            argv.forgeVer as string, getRoot(), '', getBaseURL(), false, false)
         if (resolver != null) {
             const mdl = await resolver.getModule()
             logger.info(inspect(mdl, false, null, true))

src/resolver/forge/adapter/ForgeGradle2.resolver.ts

@@ -29,9 +29,11 @@ export class ForgeGradle2Adapter extends ForgeResolver {
         relativeRoot: string,
         baseUrl: string,
         minecraftVersion: MinecraftVersion,
-        forgeVersion: string
+        forgeVersion: string,
+        discardOutput: boolean,
+        invalidateCache: boolean
     ) {
-        super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion)
+        super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion, discardOutput, invalidateCache)
     }
 
     public async getModule(): Promise<Module> {

src/resolver/forge/adapter/ForgeGradle3.resolver.ts

@@ -4,7 +4,7 @@ import { LoggerUtil } from '../../../util/LoggerUtil'
 import { VersionUtil } from '../../../util/versionutil'
 import { Module, Type } from 'helios-distribution-types'
 import { LibRepoStructure } from '../../../structure/repo/LibRepo.struct'
-import { pathExists, remove, mkdirs, copy, writeFile, readFile, lstat, move, writeJson } from 'fs-extra'
+import { pathExists, remove, mkdirs, copy, writeFile, readFile, lstat, writeJson } from 'fs-extra'
 import { join, basename, dirname } from 'path'
 import { spawn } from 'child_process'
 import { JavaUtil } from '../../../util/java/javautil'
@@ -42,9 +42,11 @@ export class ForgeGradle3Adapter extends ForgeResolver {
         relativeRoot: string,
         baseUrl: string,
         minecraftVersion: MinecraftVersion,
-        forgeVersion: string
+        forgeVersion: string,
+        discardOutput: boolean,
+        invalidateCache: boolean
     ) {
-        super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion)
+        super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion, discardOutput, invalidateCache)
         this.configure()
     }
 
@@ -198,57 +200,72 @@ export class ForgeGradle3Adapter extends ForgeResolver {
 
     private async processWithInstaller(installerPath: string): Promise<Module> {
 
-        const workDir = this.repoStructure.getWorkDirectory()
+        let doInstall = true
-        if (await pathExists(workDir)) {
+        // Check cache.
-            await remove(workDir)
+        const cacheDir = this.repoStructure.getForgeCacheDirectory(this.artifactVersion)
+        if (await pathExists(cacheDir)) {
+            if(this.invalidateCache) {
+                ForgeGradle3Adapter.logger.info(`Removing existing cache ${cacheDir}..`)
+                await remove(cacheDir)
+            } else {
+                // Use cache.
+                doInstall = false
+                ForgeGradle3Adapter.logger.info(`Using cached results at ${cacheDir}.`)
             }
+        } else {
+            await mkdirs(cacheDir)
+        }
+        const installerOutputDir = cacheDir
 
-        await mkdirs(workDir)
+        if(doInstall) {
-
+            const workingInstaller = join(installerOutputDir, basename(installerPath))
-        const workingInstaller = join(workDir, basename(installerPath))
 
             await copy(installerPath, workingInstaller)
     
             // Required for the installer to function.
-        await writeFile(join(workDir, 'launcher_profiles.json'), JSON.stringify({}))
+            await writeFile(join(installerOutputDir, 'launcher_profiles.json'), JSON.stringify({}))
     
             ForgeGradle3Adapter.logger.debug('Spawning forge installer')
     
             ForgeGradle3Adapter.logger.info('============== [ IMPORTANT ] ==============')
             ForgeGradle3Adapter.logger.info('When the installer opens please set the client installation directory to:')
-        ForgeGradle3Adapter.logger.info(workDir)
+            ForgeGradle3Adapter.logger.info(installerOutputDir)
             ForgeGradle3Adapter.logger.info('===========================================')
     
             await this.executeInstaller(workingInstaller)
     
             ForgeGradle3Adapter.logger.debug('Installer finished, beginning processing..')
+        }
 
         ForgeGradle3Adapter.logger.debug('Processing Version Manifest')
-        const versionManifestTuple = await this.processVersionManifest()
+        const versionManifestTuple = await this.processVersionManifest(installerOutputDir)
         const versionManifest = versionManifestTuple[0] as VersionManifestFG3
 
         ForgeGradle3Adapter.logger.debug('Processing generated forge files.')
-        const forgeModule = await this.processForgeModule(versionManifest)
+        const forgeModule = await this.processForgeModule(versionManifest, installerOutputDir)
 
         // Attach version.json module.
         forgeModule.subModules?.unshift(versionManifestTuple[1] as Module)
 
         ForgeGradle3Adapter.logger.debug('Processing Libraries')
-        const libs = await this.processLibraries(versionManifest)
+        const libs = await this.processLibraries(versionManifest, installerOutputDir)
 
         forgeModule.subModules = forgeModule.subModules?.concat(libs)
 
-        await remove(workDir)
+        if(this.discardOutput) {
+            ForgeGradle3Adapter.logger.info(`Removing installer output at ${installerOutputDir}..`)
+            await remove(installerOutputDir)
+            ForgeGradle3Adapter.logger.info('Removed successfully.')
+        }
 
         return forgeModule
 
     }
 
-    private async processVersionManifest(): Promise<[VersionManifestFG3, Module]> {
+    private async processVersionManifest(installerOutputDir: string): Promise<[VersionManifestFG3, Module]> {
-        const workDir = this.repoStructure.getWorkDirectory()
         const versionRepo = this.repoStructure.getVersionRepoStruct()
         const versionName = versionRepo.getFileName(this.minecraftVersion, this.forgeVersion)
-        const versionManifestPath = join(workDir, 'versions', versionName, `${versionName}.json`)
+        const versionManifestPath = join(installerOutputDir, 'versions', versionName, `${versionName}.json`)
 
         const versionManifestBuf = await readFile(versionManifestPath)
         const versionManifest = JSON.parse(versionManifestBuf.toString()) as VersionManifestFG3
@@ -269,14 +286,14 @@ export class ForgeGradle3Adapter extends ForgeResolver {
             this.forgeVersion
         )
 
-        await move(versionManifestPath, destination, {overwrite: true})
+        await copy(versionManifestPath, destination, {overwrite: true})
 
         return [versionManifest, versionManifestModule]
     }
 
-    private async processForgeModule(versionManifest: VersionManifestFG3): Promise<Module> {
+    private async processForgeModule(versionManifest: VersionManifestFG3, installerOutputDir: string): Promise<Module> {
 
-        const libDir = join(this.repoStructure.getWorkDirectory(), 'libraries')
+        const libDir = join(installerOutputDir, 'libraries')
         
         if(this.wildcardsInUse) {
             if(this.wildcardsInUse.indexOf(ForgeGradle3Adapter.WILDCARD_MCP_VERSION) > -1) {
@@ -349,7 +366,7 @@ export class ForgeGradle3Adapter extends ForgeResolver {
                         _classifier
                     )
 
-                    await move(targetLocalPath, destination, {overwrite: true})
+                    await copy(targetLocalPath, destination, {overwrite: true})
 
                     located = true
                     break classifierLoop
@@ -371,9 +388,9 @@ export class ForgeGradle3Adapter extends ForgeResolver {
         return forgeModule
     }
 
-    private async processLibraries(manifest: VersionManifestFG3): Promise<Module[]> {
+    private async processLibraries(manifest: VersionManifestFG3, installerOutputDir: string): Promise<Module[]> {
 
-        const libDir = join(this.repoStructure.getWorkDirectory(), 'libraries')
+        const libDir = join(installerOutputDir, 'libraries')
         const libRepo = this.repoStructure.getLibRepoStruct()
 
         const mdls: Module[] = []
@@ -416,7 +433,7 @@ export class ForgeGradle3Adapter extends ForgeResolver {
                     components.extension
                 )
 
-                await move(targetLocalPath, destination, {overwrite: true})
+                await copy(targetLocalPath, destination, {overwrite: true})
 
             }
         }

src/resolver/forge/forge.resolver.ts

@@ -19,7 +19,9 @@ export abstract class ForgeResolver extends BaseResolver {
         relativeRoot: string,
         baseUrl: string,
         protected minecraftVersion: MinecraftVersion,
-        protected forgeVersion: string
+        protected forgeVersion: string,
+        protected discardOutput: boolean,
+        protected invalidateCache: boolean
     ) {
         super(absoluteRoot, relativeRoot, baseUrl)
         this.repoStructure = new RepoStructure(absoluteRoot, relativeRoot)

src/structure/repo/Repo.struct.ts

@@ -1,3 +1,4 @@
+import { mkdirs } from 'fs-extra'
 import { join } from 'path'
 import { BaseFileStructure } from '../BaseFileStructure'
 import { LibRepoStructure } from './LibRepo.struct'
@@ -25,6 +26,7 @@ export class RepoStructure extends BaseFileStructure {
         super.init()
         await this.libRepoStruct.init()
         await this.versionRepoStruct.init()
+        await mkdirs(this.getCacheDirectory())
     }
 
     public getLibRepoStruct(): LibRepoStructure {
@@ -43,4 +45,12 @@ export class RepoStructure extends BaseFileStructure {
         return join(this.absoluteRoot, 'work')
     }
 
+    public getCacheDirectory(): string {
+        return join(this.absoluteRoot, 'cache')
+    }
+
+    public getForgeCacheDirectory(artifactVersion: string): string {
+        return join(this.getCacheDirectory(), 'forge', artifactVersion)
+    }
+
 }

src/structure/spec_model/Distribution.struct.ts

@@ -18,9 +18,11 @@ export class DistributionStructure implements SpecModelStructure<Distribution> {
 
     constructor(
         private absoluteRoot: string,
-        private baseUrl: string
+        private baseUrl: string,
+        discardOutput: boolean,
+        invalidateCache: boolean
     ) {
-        this.serverStruct = new ServerStructure(this.absoluteRoot, this.baseUrl)
+        this.serverStruct = new ServerStructure(this.absoluteRoot, this.baseUrl, discardOutput, invalidateCache)
         this.metaPath = join(this.absoluteRoot, 'meta')
     }
 

src/structure/spec_model/Server.struct.ts

@@ -18,7 +18,9 @@ export class ServerStructure extends BaseModelStructure<Server> {
 
     constructor(
         absoluteRoot: string,
-        baseUrl: string
+        baseUrl: string,
+        private discardOutput: boolean,
+        private invalidateCache: boolean
     ) {
         super(absoluteRoot, '', 'servers', baseUrl)
     }
@@ -133,7 +135,9 @@ export class ServerStructure extends BaseModelStructure<Server> {
                         serverMeta.forge.version,
                         dirname(this.containerDirectory),
                         '',
-                        this.baseUrl
+                        this.baseUrl,
+                        this.discardOutput,
+                        this.invalidateCache
                     )
 
                     // Resolve forge

src/util/VersionSegmentedRegistry.ts

@@ -24,11 +24,13 @@ export class VersionSegmentedRegistry {
         forgeVersion: string,
         absoluteRoot: string,
         relativeRoot: string,
-        baseURL: string
+        baseURL: string,
+        discardOutput: boolean,
+        invalidateCache: boolean
     ): ForgeResolver {
         for (const impl of VersionSegmentedRegistry.FORGE_ADAPTER_IMPL) {
             if (impl.isForVersion(minecraftVersion, forgeVersion)) {
-                return new impl(absoluteRoot, relativeRoot, baseURL, minecraftVersion, forgeVersion)
+                return new impl(absoluteRoot, relativeRoot, baseURL, minecraftVersion, forgeVersion, discardOutput, invalidateCache)
             }
         }
         throw new Error(`No forge resolver found for Minecraft ${minecraftVersion}!`)