oier/patatapack2-distribution
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add output caching for Forge 1.13+.

Browse Source 
CHANGES
 - Forge installer is no longer stored to the work directory. It is now stored in a cache folder corresponding to the artifact version.
 - Installer output is now cached by default
 - Added options to generate distro.
   - --discardOutput Delete cached output after it is no longer required. May be useful if disk space is limited.
   - --invalidateCache Invalidate and delete existing caches as they are encountered. Requires fresh cache generation.
   - Both options are false by default.
 - To invalide a single version, manually delete the folder.
 - Old functionality is essentially g distro --discardOutput --invalidateCache.
This commit is contained in:
Daniel Scalzi
2021-03-20 16:28:12 -04:00
parent 2540ca383e
commit 3f90a22972
9 changed files with 126 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/index.ts
View File

@@ -51,6 +51,26 @@ function installLocalOption(yargs: yargs.Argv): yargs.Argv {
})
}
function discardOutputOption(yargs: yargs.Argv): yargs.Argv {
return yargs.option('discardOutput', {
describe: 'Delete cached output after it is no longer required. May be useful if disk space is limited.',
type: 'boolean',
demandOption: false,
global: false,
default: false
})
}
function invalidateCacheOption(yargs: yargs.Argv): yargs.Argv {
return yargs.option('invalidateCache', {
describe: 'Invalidate and delete existing caches as they are encountered. Requires fresh cache generation.',
type: 'boolean',
demandOption: false,
global: false,
default: false
})
}
// function rootOption(yargs: yargs.Argv) {
// return yargs.option('root', {
// describe: 'File structure root.',
@@ -111,7 +131,7 @@ const initRootCommand: yargs.CommandModule = {
logger.debug('Invoked init root.')
try {
await generateSchemas(argv.root as string)
await new DistributionStructure(argv.root as string, '').init()
await new DistributionStructure(argv.root as string, '', false, false).init()
logger.info(`Successfully created new root at ${argv.root}`)
} catch (error) {
logger.error(`Failed to init new root at ${argv.root}`, error)
@@ -179,7 +199,7 @@ const generateServerCommand: yargs.CommandModule = {
}
}
const serverStruct = new ServerStructure(argv.root as string, getBaseURL())
const serverStruct = new ServerStructure(argv.root as string, getBaseURL(), false, false)
serverStruct.createServer(
argv.id as string,
minecraftVersion,
@@ -197,6 +217,8 @@ const generateDistroCommand: yargs.CommandModule = {
describe: 'Generate a distribution index from the root file structure.',
builder: (yargs) => {
yargs = installLocalOption(yargs)
yargs = discardOutputOption(yargs)
yargs = invalidateCacheOption(yargs)
yargs = namePositional(yargs)
return yargs
},
@@ -209,9 +231,13 @@ const generateDistroCommand: yargs.CommandModule = {
logger.debug(`Root set to ${argv.root}`)
logger.debug(`Base Url set to ${argv.baseUrl}`)
logger.debug(`Install option set to ${argv.installLocal}`)
logger.debug(`Discard Output option set to ${argv.discardOutput}`)
logger.debug(`Invalidate Cache option set to ${argv.invalidateCache}`)
logger.debug(`Invoked generate distro name ${finalName}.`)
const doLocalInstall = argv.installLocal as boolean
const discardOutput = argv.discardOutput as boolean ?? false
const invalidateCache = argv.invalidateCache as boolean ?? false
const heliosDataFolder = getHeliosDataFolder()
if(doLocalInstall && heliosDataFolder == null) {
logger.error('You MUST specify HELIOS_DATA_FOLDER in your .env when using the --installLocal option.')
@@ -219,7 +245,7 @@ const generateDistroCommand: yargs.CommandModule = {
}
try {
const distributionStruct = new DistributionStructure(argv.root as string, argv.baseUrl as string)
const distributionStruct = new DistributionStructure(argv.root as string, argv.baseUrl as string, discardOutput, invalidateCache)
const distro = await distributionStruct.getSpecModel()
const distroOut = JSON.stringify(distro, null, 2)
const distroPath = resolvePath(argv.root as string, finalName)
@@ -333,7 +359,7 @@ const testCommand: yargs.CommandModule = {
logger.info(process.cwd())
const mcVer = new MinecraftVersion(argv.mcVer as string)
const resolver = VersionSegmentedRegistry.getForgeResolver(mcVer,
argv.forgeVer as string, getRoot(), '', getBaseURL())
argv.forgeVer as string, getRoot(), '', getBaseURL(), false, false)
if (resolver != null) {
const mdl = await resolver.getModule()
logger.info(inspect(mdl, false, null, true))

6
src/resolver/forge/adapter/ForgeGradle2.resolver.ts
View File

@@ -29,9 +29,11 @@ export class ForgeGradle2Adapter extends ForgeResolver {
relativeRoot: string,
baseUrl: string,
minecraftVersion: MinecraftVersion,
forgeVersion: string
forgeVersion: string,
discardOutput: boolean,
invalidateCache: boolean
) {
super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion)
super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion, discardOutput, invalidateCache)
}
public async getModule(): Promise<Module> {

97
src/resolver/forge/adapter/ForgeGradle3.resolver.ts
View File

@@ -4,7 +4,7 @@ import { LoggerUtil } from '../../../util/LoggerUtil'
import { VersionUtil } from '../../../util/versionutil'
import { Module, Type } from 'helios-distribution-types'
import { LibRepoStructure } from '../../../structure/repo/LibRepo.struct'
import { pathExists, remove, mkdirs, copy, writeFile, readFile, lstat, move, writeJson } from 'fs-extra'
import { pathExists, remove, mkdirs, copy, writeFile, readFile, lstat, writeJson } from 'fs-extra'
import { join, basename, dirname } from 'path'
import { spawn } from 'child_process'
import { JavaUtil } from '../../../util/java/javautil'
@@ -42,9 +42,11 @@ export class ForgeGradle3Adapter extends ForgeResolver {
relativeRoot: string,
baseUrl: string,
minecraftVersion: MinecraftVersion,
forgeVersion: string
forgeVersion: string,
discardOutput: boolean,
invalidateCache: boolean
) {
super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion)
super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, forgeVersion, discardOutput, invalidateCache)
this.configure()
}
@@ -198,57 +200,72 @@ export class ForgeGradle3Adapter extends ForgeResolver {
private async processWithInstaller(installerPath: string): Promise<Module> {
const workDir = this.repoStructure.getWorkDirectory()
if (await pathExists(workDir)) {
await remove(workDir)
let doInstall = true
// Check cache.
const cacheDir = this.repoStructure.getForgeCacheDirectory(this.artifactVersion)
if (await pathExists(cacheDir)) {
if(this.invalidateCache) {
ForgeGradle3Adapter.logger.info(`Removing existing cache ${cacheDir}..`)
await remove(cacheDir)
} else {
// Use cache.
doInstall = false
ForgeGradle3Adapter.logger.info(`Using cached results at ${cacheDir}.`)
}
} else {
await mkdirs(cacheDir)
}
const installerOutputDir = cacheDir
if(doInstall) {
const workingInstaller = join(installerOutputDir, basename(installerPath))
await copy(installerPath, workingInstaller)
// Required for the installer to function.
await writeFile(join(installerOutputDir, 'launcher_profiles.json'), JSON.stringify({}))
ForgeGradle3Adapter.logger.debug('Spawning forge installer')
ForgeGradle3Adapter.logger.info('============== [ IMPORTANT ] ==============')
ForgeGradle3Adapter.logger.info('When the installer opens please set the client installation directory to:')
ForgeGradle3Adapter.logger.info(installerOutputDir)
ForgeGradle3Adapter.logger.info('===========================================')
await this.executeInstaller(workingInstaller)
ForgeGradle3Adapter.logger.debug('Installer finished, beginning processing..')
}
await mkdirs(workDir)
const workingInstaller = join(workDir, basename(installerPath))
await copy(installerPath, workingInstaller)
// Required for the installer to function.
await writeFile(join(workDir, 'launcher_profiles.json'), JSON.stringify({}))
ForgeGradle3Adapter.logger.debug('Spawning forge installer')
ForgeGradle3Adapter.logger.info('============== [ IMPORTANT ] ==============')
ForgeGradle3Adapter.logger.info('When the installer opens please set the client installation directory to:')
ForgeGradle3Adapter.logger.info(workDir)
ForgeGradle3Adapter.logger.info('===========================================')
await this.executeInstaller(workingInstaller)
ForgeGradle3Adapter.logger.debug('Installer finished, beginning processing..')
ForgeGradle3Adapter.logger.debug('Processing Version Manifest')
const versionManifestTuple = await this.processVersionManifest()
const versionManifestTuple = await this.processVersionManifest(installerOutputDir)
const versionManifest = versionManifestTuple[0] as VersionManifestFG3
ForgeGradle3Adapter.logger.debug('Processing generated forge files.')
const forgeModule = await this.processForgeModule(versionManifest)
const forgeModule = await this.processForgeModule(versionManifest, installerOutputDir)
// Attach version.json module.
forgeModule.subModules?.unshift(versionManifestTuple[1] as Module)
ForgeGradle3Adapter.logger.debug('Processing Libraries')
const libs = await this.processLibraries(versionManifest)
const libs = await this.processLibraries(versionManifest, installerOutputDir)
forgeModule.subModules = forgeModule.subModules?.concat(libs)
await remove(workDir)
if(this.discardOutput) {
ForgeGradle3Adapter.logger.info(`Removing installer output at ${installerOutputDir}..`)
await remove(installerOutputDir)
ForgeGradle3Adapter.logger.info('Removed successfully.')
}
return forgeModule
}
private async processVersionManifest(): Promise<[VersionManifestFG3, Module]> {
const workDir = this.repoStructure.getWorkDirectory()
private async processVersionManifest(installerOutputDir: string): Promise<[VersionManifestFG3, Module]> {
const versionRepo = this.repoStructure.getVersionRepoStruct()
const versionName = versionRepo.getFileName(this.minecraftVersion, this.forgeVersion)
const versionManifestPath = join(workDir, 'versions', versionName, `${versionName}.json`)
const versionManifestPath = join(installerOutputDir, 'versions', versionName, `${versionName}.json`)
const versionManifestBuf = await readFile(versionManifestPath)
const versionManifest = JSON.parse(versionManifestBuf.toString()) as VersionManifestFG3
@@ -269,14 +286,14 @@ export class ForgeGradle3Adapter extends ForgeResolver {
this.forgeVersion
)
await move(versionManifestPath, destination, {overwrite: true})
await copy(versionManifestPath, destination, {overwrite: true})
return [versionManifest, versionManifestModule]
}
private async processForgeModule(versionManifest: VersionManifestFG3): Promise<Module> {
private async processForgeModule(versionManifest: VersionManifestFG3, installerOutputDir: string): Promise<Module> {
const libDir = join(this.repoStructure.getWorkDirectory(), 'libraries')
const libDir = join(installerOutputDir, 'libraries')
if(this.wildcardsInUse) {
if(this.wildcardsInUse.indexOf(ForgeGradle3Adapter.WILDCARD_MCP_VERSION) > -1) {
@@ -349,7 +366,7 @@ export class ForgeGradle3Adapter extends ForgeResolver {
_classifier
)
await move(targetLocalPath, destination, {overwrite: true})
await copy(targetLocalPath, destination, {overwrite: true})
located = true
break classifierLoop
@@ -371,9 +388,9 @@ export class ForgeGradle3Adapter extends ForgeResolver {
return forgeModule
}
private async processLibraries(manifest: VersionManifestFG3): Promise<Module[]> {
private async processLibraries(manifest: VersionManifestFG3, installerOutputDir: string): Promise<Module[]> {
const libDir = join(this.repoStructure.getWorkDirectory(), 'libraries')
const libDir = join(installerOutputDir, 'libraries')
const libRepo = this.repoStructure.getLibRepoStruct()
const mdls: Module[] = []
@@ -416,7 +433,7 @@ export class ForgeGradle3Adapter extends ForgeResolver {
components.extension
)
await move(targetLocalPath, destination, {overwrite: true})
await copy(targetLocalPath, destination, {overwrite: true})
}
}

4
src/resolver/forge/forge.resolver.ts
View File

@@ -19,7 +19,9 @@ export abstract class ForgeResolver extends BaseResolver {
relativeRoot: string,
baseUrl: string,
protected minecraftVersion: MinecraftVersion,
protected forgeVersion: string
protected forgeVersion: string,
protected discardOutput: boolean,
protected invalidateCache: boolean
) {
super(absoluteRoot, relativeRoot, baseUrl)
this.repoStructure = new RepoStructure(absoluteRoot, relativeRoot)

10
src/structure/repo/Repo.struct.ts
View File

@@ -1,3 +1,4 @@
import { mkdirs } from 'fs-extra'
import { join } from 'path'
import { BaseFileStructure } from '../BaseFileStructure'
import { LibRepoStructure } from './LibRepo.struct'
@@ -25,6 +26,7 @@ export class RepoStructure extends BaseFileStructure {
super.init()
await this.libRepoStruct.init()
await this.versionRepoStruct.init()
await mkdirs(this.getCacheDirectory())
}
public getLibRepoStruct(): LibRepoStructure {
@@ -43,4 +45,12 @@ export class RepoStructure extends BaseFileStructure {
return join(this.absoluteRoot, 'work')
}
public getCacheDirectory(): string {
return join(this.absoluteRoot, 'cache')
}
public getForgeCacheDirectory(artifactVersion: string): string {
return join(this.getCacheDirectory(), 'forge', artifactVersion)
}
}

6
src/structure/spec_model/Distribution.struct.ts
View File

@@ -18,9 +18,11 @@ export class DistributionStructure implements SpecModelStructure<Distribution> {
constructor(
private absoluteRoot: string,
private baseUrl: string
private baseUrl: string,
discardOutput: boolean,
invalidateCache: boolean
) {
this.serverStruct = new ServerStructure(this.absoluteRoot, this.baseUrl)
this.serverStruct = new ServerStructure(this.absoluteRoot, this.baseUrl, discardOutput, invalidateCache)
this.metaPath = join(this.absoluteRoot, 'meta')
}

8
src/structure/spec_model/Server.struct.ts
View File

@@ -18,7 +18,9 @@ export class ServerStructure extends BaseModelStructure<Server> {
constructor(
absoluteRoot: string,
baseUrl: string
baseUrl: string,
private discardOutput: boolean,
private invalidateCache: boolean
) {
super(absoluteRoot, '', 'servers', baseUrl)
}
@@ -133,7 +135,9 @@ export class ServerStructure extends BaseModelStructure<Server> {
serverMeta.forge.version,
dirname(this.containerDirectory),
'',
this.baseUrl
this.baseUrl,
this.discardOutput,
this.invalidateCache
)
// Resolve forge

6
src/util/VersionSegmentedRegistry.ts
View File

@@ -24,11 +24,13 @@ export class VersionSegmentedRegistry {
forgeVersion: string,
absoluteRoot: string,
relativeRoot: string,
baseURL: string
baseURL: string,
discardOutput: boolean,
invalidateCache: boolean
): ForgeResolver {
for (const impl of VersionSegmentedRegistry.FORGE_ADAPTER_IMPL) {
if (impl.isForVersion(minecraftVersion, forgeVersion)) {
return new impl(absoluteRoot, relativeRoot, baseURL, minecraftVersion, forgeVersion)
return new impl(absoluteRoot, relativeRoot, baseURL, minecraftVersion, forgeVersion, discardOutput, invalidateCache)
}
}
throw new Error(`No forge resolver found for Minecraft ${minecraftVersion}!`)