diff --git a/README.md b/README.md index 89ee094..e952607 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,14 @@ Options: * OPTIONAL (default: false) * This is useful to easily test the new distribution.json in dev mode on Helios. * Tip: Set name to `dev_distribution` when using this option. +* `--discardOutput` Delete cached output after it is no longer required. May be useful if disk space is limited. + * OPTIONAL (default: false) +* `--invalidateCache` Invalidate and delete existing caches as they are encountered. Requires fresh cache generation. + * OPTIONAL (default: false) + +#### Notes + +As of Forge 1.13, the installer must be run to generate required files. The installer output is cached by default. This is done to speed up subsequent builds and allow Nebula to be run as a CI job. Options are provided to discard installer output (no caching) and invalidate caches (delete cached output and require fresh generation). To invalidate only a single version cache, manually delete the cached folder. > > Example Usage diff --git a/src/index.ts b/src/index.ts index 5e59e55..a2ea641 100644 --- a/src/index.ts +++ b/src/index.ts @@ -51,6 +51,26 @@ function installLocalOption(yargs: yargs.Argv): yargs.Argv { }) } +function discardOutputOption(yargs: yargs.Argv): yargs.Argv { + return yargs.option('discardOutput', { + describe: 'Delete cached output after it is no longer required. May be useful if disk space is limited.', + type: 'boolean', + demandOption: false, + global: false, + default: false + }) +} + +function invalidateCacheOption(yargs: yargs.Argv): yargs.Argv { + return yargs.option('invalidateCache', { + describe: 'Invalidate and delete existing caches as they are encountered. Requires fresh cache generation.', + type: 'boolean', + demandOption: false, + global: false, + default: false + }) +} + // function rootOption(yargs: yargs.Argv) { // return yargs.option('root', { // describe: 'File structure root.', @@ -111,7 +131,7 @@ const initRootCommand: yargs.CommandModule = { logger.debug('Invoked init root.') try { await generateSchemas(argv.root as string) - await new DistributionStructure(argv.root as string, '').init() + await new DistributionStructure(argv.root as string, '', false, false).init() logger.info(`Successfully created new root at ${argv.root}`) } catch (error) { logger.error(`Failed to init new root at ${argv.root}`, error) @@ -179,7 +199,7 @@ const generateServerCommand: yargs.CommandModule = { } } - const serverStruct = new ServerStructure(argv.root as string, getBaseURL()) + const serverStruct = new ServerStructure(argv.root as string, getBaseURL(), false, false) serverStruct.createServer( argv.id as string, minecraftVersion, @@ -197,6 +217,8 @@ const generateDistroCommand: yargs.CommandModule = { describe: 'Generate a distribution index from the root file structure.', builder: (yargs) => { yargs = installLocalOption(yargs) + yargs = discardOutputOption(yargs) + yargs = invalidateCacheOption(yargs) yargs = namePositional(yargs) return yargs }, @@ -209,9 +231,13 @@ const generateDistroCommand: yargs.CommandModule = { logger.debug(`Root set to ${argv.root}`) logger.debug(`Base Url set to ${argv.baseUrl}`) logger.debug(`Install option set to ${argv.installLocal}`) + logger.debug(`Discard Output option set to ${argv.discardOutput}`) + logger.debug(`Invalidate Cache option set to ${argv.invalidateCache}`) logger.debug(`Invoked generate distro name ${finalName}.`) const doLocalInstall = argv.installLocal as boolean + const discardOutput = argv.discardOutput as boolean ?? false + const invalidateCache = argv.invalidateCache as boolean ?? false const heliosDataFolder = getHeliosDataFolder() if(doLocalInstall && heliosDataFolder == null) { logger.error('You MUST specify HELIOS_DATA_FOLDER in your .env when using the --installLocal option.') @@ -219,7 +245,7 @@ const generateDistroCommand: yargs.CommandModule = { } try { - const distributionStruct = new DistributionStructure(argv.root as string, argv.baseUrl as string) + const distributionStruct = new DistributionStructure(argv.root as string, argv.baseUrl as string, discardOutput, invalidateCache) const distro = await distributionStruct.getSpecModel() const distroOut = JSON.stringify(distro, null, 2) const distroPath = resolvePath(argv.root as string, finalName) @@ -333,7 +359,7 @@ const testCommand: yargs.CommandModule = { logger.info(process.cwd()) const mcVer = new MinecraftVersion(argv.mcVer as string) const resolver = VersionSegmentedRegistry.getForgeResolver(mcVer, - argv.forgeVer as string, getRoot(), '', getBaseURL()) + argv.forgeVer as string, getRoot(), '', getBaseURL(), false, false) if (resolver != null) { const mdl = await resolver.getModule() logger.info(inspect(mdl, false, null, true)) diff --git a/src/resolver/forge/adapter/ForgeGradle2.resolver.ts b/src/resolver/forge/adapter/ForgeGradle2.resolver.ts index 21fb738..81cb77e 100644 --- a/src/resolver/forge/adapter/ForgeGradle2.resolver.ts +++ b/src/resolver/forge/adapter/ForgeGradle2.resolver.ts @@ -29,9 +29,11 @@ export class ForgeGradle2Adapter extends ForgeResolver { relativeRoot: string, baseUrl: string, minecraftVersion: MinecraftVersion, - forgeVersion: string + forgeVersion: string, + discardOutput: boolean, + invalidateCache: boolean ) { - super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion) + super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion, discardOutput, invalidateCache) } public async getModule(): Promise { diff --git a/src/resolver/forge/adapter/ForgeGradle3.resolver.ts b/src/resolver/forge/adapter/ForgeGradle3.resolver.ts index 71721ef..8bef0ed 100644 --- a/src/resolver/forge/adapter/ForgeGradle3.resolver.ts +++ b/src/resolver/forge/adapter/ForgeGradle3.resolver.ts @@ -4,7 +4,7 @@ import { LoggerUtil } from '../../../util/LoggerUtil' import { VersionUtil } from '../../../util/versionutil' import { Module, Type } from 'helios-distribution-types' import { LibRepoStructure } from '../../../structure/repo/LibRepo.struct' -import { pathExists, remove, mkdirs, copy, writeFile, readFile, lstat, move, writeJson } from 'fs-extra' +import { pathExists, remove, mkdirs, copy, writeFile, readFile, lstat, writeJson } from 'fs-extra' import { join, basename, dirname } from 'path' import { spawn } from 'child_process' import { JavaUtil } from '../../../util/java/javautil' @@ -42,9 +42,11 @@ export class ForgeGradle3Adapter extends ForgeResolver { relativeRoot: string, baseUrl: string, minecraftVersion: MinecraftVersion, - forgeVersion: string + forgeVersion: string, + discardOutput: boolean, + invalidateCache: boolean ) { - super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion) + super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion, discardOutput, invalidateCache) this.configure() } @@ -198,57 +200,72 @@ export class ForgeGradle3Adapter extends ForgeResolver { private async processWithInstaller(installerPath: string): Promise { - const workDir = this.repoStructure.getWorkDirectory() - if (await pathExists(workDir)) { - await remove(workDir) + let doInstall = true + // Check cache. + const cacheDir = this.repoStructure.getForgeCacheDirectory(this.artifactVersion) + if (await pathExists(cacheDir)) { + if(this.invalidateCache) { + ForgeGradle3Adapter.logger.info(`Removing existing cache ${cacheDir}..`) + await remove(cacheDir) + } else { + // Use cache. + doInstall = false + ForgeGradle3Adapter.logger.info(`Using cached results at ${cacheDir}.`) + } + } else { + await mkdirs(cacheDir) + } + const installerOutputDir = cacheDir + + if(doInstall) { + const workingInstaller = join(installerOutputDir, basename(installerPath)) + + await copy(installerPath, workingInstaller) + + // Required for the installer to function. + await writeFile(join(installerOutputDir, 'launcher_profiles.json'), JSON.stringify({})) + + ForgeGradle3Adapter.logger.debug('Spawning forge installer') + + ForgeGradle3Adapter.logger.info('============== [ IMPORTANT ] ==============') + ForgeGradle3Adapter.logger.info('When the installer opens please set the client installation directory to:') + ForgeGradle3Adapter.logger.info(installerOutputDir) + ForgeGradle3Adapter.logger.info('===========================================') + + await this.executeInstaller(workingInstaller) + + ForgeGradle3Adapter.logger.debug('Installer finished, beginning processing..') } - await mkdirs(workDir) - - const workingInstaller = join(workDir, basename(installerPath)) - - await copy(installerPath, workingInstaller) - - // Required for the installer to function. - await writeFile(join(workDir, 'launcher_profiles.json'), JSON.stringify({})) - - ForgeGradle3Adapter.logger.debug('Spawning forge installer') - - ForgeGradle3Adapter.logger.info('============== [ IMPORTANT ] ==============') - ForgeGradle3Adapter.logger.info('When the installer opens please set the client installation directory to:') - ForgeGradle3Adapter.logger.info(workDir) - ForgeGradle3Adapter.logger.info('===========================================') - - await this.executeInstaller(workingInstaller) - - ForgeGradle3Adapter.logger.debug('Installer finished, beginning processing..') - ForgeGradle3Adapter.logger.debug('Processing Version Manifest') - const versionManifestTuple = await this.processVersionManifest() + const versionManifestTuple = await this.processVersionManifest(installerOutputDir) const versionManifest = versionManifestTuple[0] as VersionManifestFG3 ForgeGradle3Adapter.logger.debug('Processing generated forge files.') - const forgeModule = await this.processForgeModule(versionManifest) + const forgeModule = await this.processForgeModule(versionManifest, installerOutputDir) // Attach version.json module. forgeModule.subModules?.unshift(versionManifestTuple[1] as Module) ForgeGradle3Adapter.logger.debug('Processing Libraries') - const libs = await this.processLibraries(versionManifest) + const libs = await this.processLibraries(versionManifest, installerOutputDir) forgeModule.subModules = forgeModule.subModules?.concat(libs) - await remove(workDir) + if(this.discardOutput) { + ForgeGradle3Adapter.logger.info(`Removing installer output at ${installerOutputDir}..`) + await remove(installerOutputDir) + ForgeGradle3Adapter.logger.info('Removed successfully.') + } return forgeModule } - private async processVersionManifest(): Promise<[VersionManifestFG3, Module]> { - const workDir = this.repoStructure.getWorkDirectory() + private async processVersionManifest(installerOutputDir: string): Promise<[VersionManifestFG3, Module]> { const versionRepo = this.repoStructure.getVersionRepoStruct() const versionName = versionRepo.getFileName(this.minecraftVersion, this.forgeVersion) - const versionManifestPath = join(workDir, 'versions', versionName, `${versionName}.json`) + const versionManifestPath = join(installerOutputDir, 'versions', versionName, `${versionName}.json`) const versionManifestBuf = await readFile(versionManifestPath) const versionManifest = JSON.parse(versionManifestBuf.toString()) as VersionManifestFG3 @@ -269,14 +286,14 @@ export class ForgeGradle3Adapter extends ForgeResolver { this.forgeVersion ) - await move(versionManifestPath, destination, {overwrite: true}) + await copy(versionManifestPath, destination, {overwrite: true}) return [versionManifest, versionManifestModule] } - private async processForgeModule(versionManifest: VersionManifestFG3): Promise { + private async processForgeModule(versionManifest: VersionManifestFG3, installerOutputDir: string): Promise { - const libDir = join(this.repoStructure.getWorkDirectory(), 'libraries') + const libDir = join(installerOutputDir, 'libraries') if(this.wildcardsInUse) { if(this.wildcardsInUse.indexOf(ForgeGradle3Adapter.WILDCARD_MCP_VERSION) > -1) { @@ -349,7 +366,7 @@ export class ForgeGradle3Adapter extends ForgeResolver { _classifier ) - await move(targetLocalPath, destination, {overwrite: true}) + await copy(targetLocalPath, destination, {overwrite: true}) located = true break classifierLoop @@ -371,9 +388,9 @@ export class ForgeGradle3Adapter extends ForgeResolver { return forgeModule } - private async processLibraries(manifest: VersionManifestFG3): Promise { + private async processLibraries(manifest: VersionManifestFG3, installerOutputDir: string): Promise { - const libDir = join(this.repoStructure.getWorkDirectory(), 'libraries') + const libDir = join(installerOutputDir, 'libraries') const libRepo = this.repoStructure.getLibRepoStruct() const mdls: Module[] = [] @@ -416,7 +433,7 @@ export class ForgeGradle3Adapter extends ForgeResolver { components.extension ) - await move(targetLocalPath, destination, {overwrite: true}) + await copy(targetLocalPath, destination, {overwrite: true}) } } diff --git a/src/resolver/forge/forge.resolver.ts b/src/resolver/forge/forge.resolver.ts index 19bb592..0959d12 100644 --- a/src/resolver/forge/forge.resolver.ts +++ b/src/resolver/forge/forge.resolver.ts @@ -19,7 +19,9 @@ export abstract class ForgeResolver extends BaseResolver { relativeRoot: string, baseUrl: string, protected minecraftVersion: MinecraftVersion, - protected forgeVersion: string + protected forgeVersion: string, + protected discardOutput: boolean, + protected invalidateCache: boolean ) { super(absoluteRoot, relativeRoot, baseUrl) this.repoStructure = new RepoStructure(absoluteRoot, relativeRoot) diff --git a/src/structure/repo/Repo.struct.ts b/src/structure/repo/Repo.struct.ts index 8450406..c3f8917 100644 --- a/src/structure/repo/Repo.struct.ts +++ b/src/structure/repo/Repo.struct.ts @@ -1,3 +1,4 @@ +import { mkdirs } from 'fs-extra' import { join } from 'path' import { BaseFileStructure } from '../BaseFileStructure' import { LibRepoStructure } from './LibRepo.struct' @@ -25,6 +26,7 @@ export class RepoStructure extends BaseFileStructure { super.init() await this.libRepoStruct.init() await this.versionRepoStruct.init() + await mkdirs(this.getCacheDirectory()) } public getLibRepoStruct(): LibRepoStructure { @@ -43,4 +45,12 @@ export class RepoStructure extends BaseFileStructure { return join(this.absoluteRoot, 'work') } + public getCacheDirectory(): string { + return join(this.absoluteRoot, 'cache') + } + + public getForgeCacheDirectory(artifactVersion: string): string { + return join(this.getCacheDirectory(), 'forge', artifactVersion) + } + } diff --git a/src/structure/spec_model/Distribution.struct.ts b/src/structure/spec_model/Distribution.struct.ts index 2b62ead..5213139 100644 --- a/src/structure/spec_model/Distribution.struct.ts +++ b/src/structure/spec_model/Distribution.struct.ts @@ -18,9 +18,11 @@ export class DistributionStructure implements SpecModelStructure { constructor( private absoluteRoot: string, - private baseUrl: string + private baseUrl: string, + discardOutput: boolean, + invalidateCache: boolean ) { - this.serverStruct = new ServerStructure(this.absoluteRoot, this.baseUrl) + this.serverStruct = new ServerStructure(this.absoluteRoot, this.baseUrl, discardOutput, invalidateCache) this.metaPath = join(this.absoluteRoot, 'meta') } diff --git a/src/structure/spec_model/Server.struct.ts b/src/structure/spec_model/Server.struct.ts index 1cf7426..79d9869 100644 --- a/src/structure/spec_model/Server.struct.ts +++ b/src/structure/spec_model/Server.struct.ts @@ -18,7 +18,9 @@ export class ServerStructure extends BaseModelStructure { constructor( absoluteRoot: string, - baseUrl: string + baseUrl: string, + private discardOutput: boolean, + private invalidateCache: boolean ) { super(absoluteRoot, '', 'servers', baseUrl) } @@ -133,7 +135,9 @@ export class ServerStructure extends BaseModelStructure { serverMeta.forge.version, dirname(this.containerDirectory), '', - this.baseUrl + this.baseUrl, + this.discardOutput, + this.invalidateCache ) // Resolve forge diff --git a/src/util/VersionSegmentedRegistry.ts b/src/util/VersionSegmentedRegistry.ts index 2db1aec..12a0696 100644 --- a/src/util/VersionSegmentedRegistry.ts +++ b/src/util/VersionSegmentedRegistry.ts @@ -24,11 +24,13 @@ export class VersionSegmentedRegistry { forgeVersion: string, absoluteRoot: string, relativeRoot: string, - baseURL: string + baseURL: string, + discardOutput: boolean, + invalidateCache: boolean ): ForgeResolver { for (const impl of VersionSegmentedRegistry.FORGE_ADAPTER_IMPL) { if (impl.isForVersion(minecraftVersion, forgeVersion)) { - return new impl(absoluteRoot, relativeRoot, baseURL, minecraftVersion, forgeVersion) + return new impl(absoluteRoot, relativeRoot, baseURL, minecraftVersion, forgeVersion, discardOutput, invalidateCache) } } throw new Error(`No forge resolver found for Minecraft ${minecraftVersion}!`)