oier/patatapack2-distribution
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add output caching for Forge 1.13+.

Browse Source 
CHANGES
 - Forge installer is no longer stored to the work directory. It is now stored in a cache folder corresponding to the artifact version.
 - Installer output is now cached by default
 - Added options to generate distro.
   - --discardOutput Delete cached output after it is no longer required. May be useful if disk space is limited.
   - --invalidateCache Invalidate and delete existing caches as they are encountered. Requires fresh cache generation.
   - Both options are false by default.
 - To invalide a single version, manually delete the folder.
 - Old functionality is essentially g distro --discardOutput --invalidateCache.
This commit is contained in:
Daniel Scalzi
2021-03-20 16:28:12 -04:00
parent 2540ca383e
commit 3f90a22972
9 changed files with 126 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@@ -138,6 +138,14 @@ Options:
* OPTIONAL (default: false)
* This is useful to easily test the new distribution.json in dev mode on Helios.
* Tip: Set name to `dev_distribution` when using this option.
* `--discardOutput` Delete cached output after it is no longer required. May be useful if disk space is limited.
* OPTIONAL (default: false)
* `--invalidateCache` Invalidate and delete existing caches as they are encountered. Requires fresh cache generation.
* OPTIONAL (default: false)
#### Notes
As of Forge 1.13, the installer must be run to generate required files. The installer output is cached by default. This is done to speed up subsequent builds and allow Nebula to be run as a CI job. Options are provided to discard installer output (no caching) and invalidate caches (delete cached output and require fresh generation). To invalidate only a single version cache, manually delete the cached folder.
>
> Example Usage

34
src/index.ts
View File

@@ -51,6 +51,26 @@ function installLocalOption(yargs: yargs.Argv): yargs.Argv {
})
}
function discardOutputOption(yargs: yargs.Argv): yargs.Argv {
return yargs.option('discardOutput', {
describe: 'Delete cached output after it is no longer required. May be useful if disk space is limited.',
type: 'boolean',
demandOption: false,
global: false,
default: false
})
}
function invalidateCacheOption(yargs: yargs.Argv): yargs.Argv {
return yargs.option('invalidateCache', {
describe: 'Invalidate and delete existing caches as they are encountered. Requires fresh cache generation.',
type: 'boolean',
demandOption: false,
global: false,
default: false
})
}
// function rootOption(yargs: yargs.Argv) {
// return yargs.option('root', {
// describe: 'File structure root.',
@@ -111,7 +131,7 @@ const initRootCommand: yargs.CommandModule = {
logger.debug('Invoked init root.')
try {
await generateSchemas(argv.root as string)
await new DistributionStructure(argv.root as string, '').init()
await new DistributionStructure(argv.root as string, '', false, false).init()
logger.info(`Successfully created new root at ${argv.root}`)
} catch (error) {
logger.error(`Failed to init new root at ${argv.root}`, error)
@@ -179,7 +199,7 @@ const generateServerCommand: yargs.CommandModule = {
}
}
const serverStruct = new ServerStructure(argv.root as string, getBaseURL())
const serverStruct = new ServerStructure(argv.root as string, getBaseURL(), false, false)
serverStruct.createServer(
argv.id as string,
minecraftVersion,
@@ -197,6 +217,8 @@ const generateDistroCommand: yargs.CommandModule = {
describe: 'Generate a distribution index from the root file structure.',
builder: (yargs) => {
yargs = installLocalOption(yargs)
yargs = discardOutputOption(yargs)
yargs = invalidateCacheOption(yargs)
yargs = namePositional(yargs)
return yargs
},
@@ -209,9 +231,13 @@ const generateDistroCommand: yargs.CommandModule = {
logger.debug(`Root set to ${argv.root}`)
logger.debug(`Base Url set to ${argv.baseUrl}`)
logger.debug(`Install option set to ${argv.installLocal}`)
logger.debug(`Discard Output option set to ${argv.discardOutput}`)
logger.debug(`Invalidate Cache option set to ${argv.invalidateCache}`)
logger.debug(`Invoked generate distro name ${finalName}.`)
const doLocalInstall = argv.installLocal as boolean
const discardOutput = argv.discardOutput as boolean ?? false
const invalidateCache = argv.invalidateCache as boolean ?? false
const heliosDataFolder = getHeliosDataFolder()
if(doLocalInstall && heliosDataFolder == null) {
logger.error('You MUST specify HELIOS_DATA_FOLDER in your .env when using the --installLocal option.')
@@ -219,7 +245,7 @@ const generateDistroCommand: yargs.CommandModule = {
}
try {
const distributionStruct = new DistributionStructure(argv.root as string, argv.baseUrl as string)
const distributionStruct = new DistributionStructure(argv.root as string, argv.baseUrl as string, discardOutput, invalidateCache)
const distro = await distributionStruct.getSpecModel()
const distroOut = JSON.stringify(distro, null, 2)
const distroPath = resolvePath(argv.root as string, finalName)
@@ -333,7 +359,7 @@ const testCommand: yargs.CommandModule = {
logger.info(process.cwd())
const mcVer = new MinecraftVersion(argv.mcVer as string)
const resolver = VersionSegmentedRegistry.getForgeResolver(mcVer,
argv.forgeVer as string, getRoot(), '', getBaseURL())
argv.forgeVer as string, getRoot(), '', getBaseURL(), false, false)
if (resolver != null) {
const mdl = await resolver.getModule()
logger.info(inspect(mdl, false, null, true))

6
src/resolver/forge/adapter/ForgeGradle2.resolver.ts
View File

@@ -29,9 +29,11 @@ export class ForgeGradle2Adapter extends ForgeResolver {
relativeRoot: string,
baseUrl: string,
minecraftVersion: MinecraftVersion,
forgeVersion: string
forgeVersion: string,
discardOutput: boolean,
invalidateCache: boolean
) {
super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion)
super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion, discardOutput, invalidateCache)
}
public async getModule(): Promise<Module> {

67
src/resolver/forge/adapter/ForgeGradle3.resolver.ts
View File

@@ -4,7 +4,7 @@ import { LoggerUtil } from '../../../util/LoggerUtil'
import { VersionUtil } from '../../../util/versionutil'
import { Module, Type } from 'helios-distribution-types'
import { LibRepoStructure } from '../../../structure/repo/LibRepo.struct'
import { pathExists, remove, mkdirs, copy, writeFile, readFile, lstat, move, writeJson } from 'fs-extra'
import { pathExists, remove, mkdirs, copy, writeFile, readFile, lstat, writeJson } from 'fs-extra'
import { join, basename, dirname } from 'path'
import { spawn } from 'child_process'
import { JavaUtil } from '../../../util/java/javautil'
@@ -42,9 +42,11 @@ export class ForgeGradle3Adapter extends ForgeResolver {
relativeRoot: string,
baseUrl: string,
minecraftVersion: MinecraftVersion,
forgeVersion: string
forgeVersion: string,
discardOutput: boolean,
invalidateCache: boolean
) {
super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion)
super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion, discardOutput, invalidateCache)
this.configure()
}
@@ -198,57 +200,72 @@ export class ForgeGradle3Adapter extends ForgeResolver {
private async processWithInstaller(installerPath: string): Promise<Module> {
const workDir = this.repoStructure.getWorkDirectory()
if (await pathExists(workDir)) {
await remove(workDir)
let doInstall = true
// Check cache.
const cacheDir = this.repoStructure.getForgeCacheDirectory(this.artifactVersion)
if (await pathExists(cacheDir)) {
if(this.invalidateCache) {
ForgeGradle3Adapter.logger.info(`Removing existing cache ${cacheDir}..`)
await remove(cacheDir)
} else {
// Use cache.
doInstall = false
ForgeGradle3Adapter.logger.info(`Using cached results at ${cacheDir}.`)
}
} else {
await mkdirs(cacheDir)
}
const installerOutputDir = cacheDir
await mkdirs(workDir)
const workingInstaller = join(workDir, basename(installerPath))
if(doInstall) {
const workingInstaller = join(installerOutputDir, basename(installerPath))
await copy(installerPath, workingInstaller)
// Required for the installer to function.
await writeFile(join(workDir, 'launcher_profiles.json'), JSON.stringify({}))
await writeFile(join(installerOutputDir, 'launcher_profiles.json'), JSON.stringify({}))
ForgeGradle3Adapter.logger.debug('Spawning forge installer')
ForgeGradle3Adapter.logger.info('============== [ IMPORTANT ] ==============')
ForgeGradle3Adapter.logger.info('When the installer opens please set the client installation directory to:')
ForgeGradle3Adapter.logger.info(workDir)
ForgeGradle3Adapter.logger.info(installerOutputDir)
ForgeGradle3Adapter.logger.info('===========================================')
await this.executeInstaller(workingInstaller)
ForgeGradle3Adapter.logger.debug('Installer finished, beginning processing..')
}
ForgeGradle3Adapter.logger.debug('Processing Version Manifest')
const versionManifestTuple = await this.processVersionManifest()
const versionManifestTuple = await this.processVersionManifest(installerOutputDir)
const versionManifest = versionManifestTuple[0] as VersionManifestFG3
ForgeGradle3Adapter.logger.debug('Processing generated forge files.')
const forgeModule = await this.processForgeModule(versionManifest)
const forgeModule = await this.processForgeModule(versionManifest, installerOutputDir)
// Attach version.json module.
forgeModule.subModules?.unshift(versionManifestTuple[1] as Module)
ForgeGradle3Adapter.logger.debug('Processing Libraries')
const libs = await this.processLibraries(versionManifest)
const libs = await this.processLibraries(versionManifest, installerOutputDir)
forgeModule.subModules = forgeModule.subModules?.concat(libs)
await remove(workDir)
if(this.discardOutput) {
ForgeGradle3Adapter.logger.info(`Removing installer output at ${installerOutputDir}..`)
await remove(installerOutputDir)
ForgeGradle3Adapter.logger.info('Removed successfully.')
}
return forgeModule
}
private async processVersionManifest(): Promise<[VersionManifestFG3, Module]> {
const workDir = this.repoStructure.getWorkDirectory()
private async processVersionManifest(installerOutputDir: string): Promise<[VersionManifestFG3, Module]> {
const versionRepo = this.repoStructure.getVersionRepoStruct()
const versionName = versionRepo.getFileName(this.minecraftVersion, this.forgeVersion)
const versionManifestPath = join(workDir, 'versions', versionName, `${versionName}.json`)
const versionManifestPath = join(installerOutputDir, 'versions', versionName, `${versionName}.json`)
const versionManifestBuf = await readFile(versionManifestPath)
const versionManifest = JSON.parse(versionManifestBuf.toString()) as VersionManifestFG3
@@ -269,14 +286,14 @@ export class ForgeGradle3Adapter extends ForgeResolver {
this.forgeVersion
)
await move(versionManifestPath, destination, {overwrite: true})
await copy(versionManifestPath, destination, {overwrite: true})
return [versionManifest, versionManifestModule]
}
private async processForgeModule(versionManifest: VersionManifestFG3): Promise<Module> {
private async processForgeModule(versionManifest: VersionManifestFG3, installerOutputDir: string): Promise<Module> {
const libDir = join(this.repoStructure.getWorkDirectory(), 'libraries')
const libDir = join(installerOutputDir, 'libraries')
if(this.wildcardsInUse) {
if(this.wildcardsInUse.indexOf(ForgeGradle3Adapter.WILDCARD_MCP_VERSION) > -1) {
@@ -349,7 +366,7 @@ export class ForgeGradle3Adapter extends ForgeResolver {
_classifier
)
await move(targetLocalPath, destination, {overwrite: true})
await copy(targetLocalPath, destination, {overwrite: true})
located = true
break classifierLoop
@@ -371,9 +388,9 @@ export class ForgeGradle3Adapter extends ForgeResolver {
return forgeModule
}
private async processLibraries(manifest: VersionManifestFG3): Promise<Module[]> {
private async processLibraries(manifest: VersionManifestFG3, installerOutputDir: string): Promise<Module[]> {
const libDir = join(this.repoStructure.getWorkDirectory(), 'libraries')
const libDir = join(installerOutputDir, 'libraries')
const libRepo = this.repoStructure.getLibRepoStruct()
const mdls: Module[] = []
@@ -416,7 +433,7 @@ export class ForgeGradle3Adapter extends ForgeResolver {
components.extension
)
await move(targetLocalPath, destination, {overwrite: true})
await copy(targetLocalPath, destination, {overwrite: true})
}
}

4
src/resolver/forge/forge.resolver.ts
View File

@@ -19,7 +19,9 @@ export abstract class ForgeResolver extends BaseResolver {
relativeRoot: string,
baseUrl: string,
protected minecraftVersion: MinecraftVersion,
protected forgeVersion: string
protected forgeVersion: string,
protected discardOutput: boolean,
protected invalidateCache: boolean
) {
super(absoluteRoot, relativeRoot, baseUrl)
this.repoStructure = new RepoStructure(absoluteRoot, relativeRoot)

10
src/structure/repo/Repo.struct.ts
View File

@@ -1,3 +1,4 @@
import { mkdirs } from 'fs-extra'
import { join } from 'path'
import { BaseFileStructure } from '../BaseFileStructure'
import { LibRepoStructure } from './LibRepo.struct'
@@ -25,6 +26,7 @@ export class RepoStructure extends BaseFileStructure {
super.init()
await this.libRepoStruct.init()
await this.versionRepoStruct.init()
await mkdirs(this.getCacheDirectory())
}
public getLibRepoStruct(): LibRepoStructure {
@@ -43,4 +45,12 @@ export class RepoStructure extends BaseFileStructure {
return join(this.absoluteRoot, 'work')
}
public getCacheDirectory(): string {
return join(this.absoluteRoot, 'cache')
}
public getForgeCacheDirectory(artifactVersion: string): string {
return join(this.getCacheDirectory(), 'forge', artifactVersion)
}
}

6
src/structure/spec_model/Distribution.struct.ts
View File

@@ -18,9 +18,11 @@ export class DistributionStructure implements SpecModelStructure<Distribution> {
constructor(
private absoluteRoot: string,
private baseUrl: string
private baseUrl: string,
discardOutput: boolean,
invalidateCache: boolean
) {
this.serverStruct = new ServerStructure(this.absoluteRoot, this.baseUrl)
this.serverStruct = new ServerStructure(this.absoluteRoot, this.baseUrl, discardOutput, invalidateCache)
this.metaPath = join(this.absoluteRoot, 'meta')
}

8
src/structure/spec_model/Server.struct.ts
View File

@@ -18,7 +18,9 @@ export class ServerStructure extends BaseModelStructure<Server> {
constructor(
absoluteRoot: string,
baseUrl: string
baseUrl: string,
private discardOutput: boolean,
private invalidateCache: boolean
) {
super(absoluteRoot, '', 'servers', baseUrl)
}
@@ -133,7 +135,9 @@ export class ServerStructure extends BaseModelStructure<Server> {
serverMeta.forge.version,
dirname(this.containerDirectory),
'',
this.baseUrl
this.baseUrl,
this.discardOutput,
this.invalidateCache
)
// Resolve forge

6
src/util/VersionSegmentedRegistry.ts
View File

@@ -24,11 +24,13 @@ export class VersionSegmentedRegistry {
forgeVersion: string,
absoluteRoot: string,
relativeRoot: string,
baseURL: string
baseURL: string,
discardOutput: boolean,
invalidateCache: boolean
): ForgeResolver {
for (const impl of VersionSegmentedRegistry.FORGE_ADAPTER_IMPL) {
if (impl.isForVersion(minecraftVersion, forgeVersion)) {
return new impl(absoluteRoot, relativeRoot, baseURL, minecraftVersion, forgeVersion)
return new impl(absoluteRoot, relativeRoot, baseURL, minecraftVersion, forgeVersion, discardOutput, invalidateCache)
}
}
throw new Error(`No forge resolver found for Minecraft ${minecraftVersion}!`)