oier/patatapack2-distribution
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for untracked files.

Browse Source 
Untracked file glob patterns are stored in the servermeta.json. See the README for detailed information.
This commit is contained in:
Daniel Scalzi
2020-09-12 22:46:50 -04:00
parent 1a19df0e93
commit 24b0923903
14 changed files with 153 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
README.md
View File

@@ -240,10 +240,51 @@ Represents the additional metadata on the server object (for a YOUR_SERVER). Sam
},
"forge": {
"version": "14.23.5.2854"
}
},
"untrackedFiles": []
}
```
Untracked files is optional. MD5 hashes will not be generated for files matching the provided glob patterns.
```json
{
"untrackedFiles": [
{
"appliesTo": ["files"],
"patterns": [
"config/*.cfg",
"config/**/*.yml"
]
}
]
}
```
In the above example, all files of type `cfg` in the config directory will be untracked. Additionally, all files of type `yml` in the config directory and its subdirectories will be untracked. You can tweak these patterns to fit your needs, this is purely an example. The patterns will only be applied to the folders specified in `appliesTo`. As an example, valid values include `files`, `forgemods`, `libraries`, etc.
```json
{
"untrackedFiles": [
{
"appliesTo": ["files"],
"patterns": [
"config/*.cfg",
"config/**/*.yml"
]
},
{
"appliesTo": ["forgemods", "litemods"],
"patterns": [
"optionalon/*.jar"
]
}
]
}
```
Another example where all `optionalon` forgemods and litemods are untracked. **Untracking mods is NOT recommended. This is an example ONLY.**
[dotenvnpm]: https://www.npmjs.com/package/dotenv
[distro.md]: https://github.com/dscalzi/HeliosLauncher/blob/master/docs/distro.md

20
package-lock.json generated
View File

@@ -186,6 +186,12 @@
"@types/node": "*"
}
},
"@types/minimatch": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
"integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==",
"dev": true
},
"@types/node": {
"version": "12.12.58",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.58.tgz",
@@ -381,14 +387,12 @@
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
"dev": true
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@@ -536,8 +540,7 @@
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
"dev": true
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
"core-util-is": {
"version": "1.0.2",
@@ -1028,9 +1031,9 @@
"dev": true
},
"helios-distribution-types": {
"version": "1.0.0-pre.1",
"resolved": "https://registry.npmjs.org/helios-distribution-types/-/helios-distribution-types-1.0.0-pre.1.tgz",
"integrity": "sha512-rqmuLoiyZTLGH0rlklRpjTvVbBTtP/NOyQqCvBPaLwpU2xUX/Vxzmt3RE8k+OgN3BAUEVJ+4jhSLgCsCXJfx1g=="
"version": "1.0.0-rc.1",
"resolved": "https://registry.npmjs.org/helios-distribution-types/-/helios-distribution-types-1.0.0-rc.1.tgz",
"integrity": "sha512-b3GNQoTDexFZ7CvW5F2vJrKbWpQZKof1uWOWT9vcvrPE91WLhzZf3UBdBOzIlSHpXiu58dZxh7eLZw+HUE7YLA=="
},
"http-cache-semantics": {
"version": "4.1.0",
@@ -1250,7 +1253,6 @@
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dev": true,
"requires": {
"brace-expansion": "^1.1.7"
}

4
package.json
View File

@@ -27,6 +27,7 @@
"homepage": "https://github.com/dscalzi/Nebula#readme",
"devDependencies": {
"@types/fs-extra": "^9.0.1",
"@types/minimatch": "^3.0.3",
"@types/node": "^12.12.58",
"@types/triple-beam": "^1.3.2",
"@types/yargs": "^15.0.5",
@@ -40,7 +41,8 @@
"dotenv": "^8.2.0",
"fs-extra": "^9.0.1",
"got": "^11.6.2",
"helios-distribution-types": "^1.0.0-pre.1",
"helios-distribution-types": "^1.0.0-rc.1",
"minimatch": "^3.0.4",
"moment": "^2.27.0",
"node-stream-zip": "^1.11.3",
"toml": "^3.0.0",

17
src/model/nebula/servermeta.ts
View File

@@ -1,5 +1,17 @@
import { Server } from 'helios-distribution-types'
export interface UntrackedFilesOption {
/**
* The subdirectory this applies to. Ex.
* [ 'files', 'forgemods' ]
*/
appliesTo: string[]
/**
* Glob patterns to match against the file.
*/
patterns: string[]
}
export interface ServerMetaOptions {
forgeVersion?: string
liteloaderVersion?: string
@@ -37,6 +49,9 @@ export function getDefaultServerMeta(id: string, version: string, options?: Serv
}
}
// Add empty untracked files.
servMeta.untrackedFiles = []
return servMeta
}
@@ -60,4 +75,6 @@ export interface ServerMeta {
version: string
}
untrackedFiles?: UntrackedFilesOption[]
}

21
src/structure/spec_model/Server.struct.ts
View File

@@ -3,7 +3,7 @@ import { Server, Module } from 'helios-distribution-types'
import { dirname, join, resolve as resolvePath } from 'path'
import { resolve as resolveUrl } from 'url'
import { VersionSegmentedRegistry } from '../../util/VersionSegmentedRegistry'
import { ServerMeta, getDefaultServerMeta, ServerMetaOptions } from '../../model/nebula/servermeta'
import { ServerMeta, getDefaultServerMeta, ServerMetaOptions, UntrackedFilesOption } from '../../model/nebula/servermeta'
import { BaseModelStructure } from './BaseModel.struct'
import { MiscFileStructure } from './module/File.struct'
import { LiteModStructure } from './module/LiteMod.struct'
@@ -60,14 +60,15 @@ export class ServerStructure extends BaseModelStructure<Server> {
options.forgeVersion,
absoluteServerRoot,
relativeServerRoot,
this.baseUrl
this.baseUrl,
[]
)
await fms.init()
serverMetaOpts.forgeVersion = options.forgeVersion
}
if (options.liteloaderVersion != null) {
const lms = new LiteModStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion)
const lms = new LiteModStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion, [])
await lms.init()
serverMetaOpts.liteloaderVersion = options.liteloaderVersion
}
@@ -75,10 +76,10 @@ export class ServerStructure extends BaseModelStructure<Server> {
const serverMeta: ServerMeta = getDefaultServerMeta(id, minecraftVersion.toString(), serverMetaOpts)
await writeFile(resolvePath(absoluteServerRoot, this.SERVER_META_FILE), JSON.stringify(serverMeta, null, 2))
const libS = new LibraryStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion)
const libS = new LibraryStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion, [])
await libS.init()
const mfs = new MiscFileStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion)
const mfs = new MiscFileStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion, [])
await mfs.init()
}
@@ -117,6 +118,7 @@ export class ServerStructure extends BaseModelStructure<Server> {
// Read server meta
const serverMeta: ServerMeta = JSON.parse(await readFile(resolvePath(absoluteServerRoot, this.SERVER_META_FILE), 'utf-8'))
const minecraftVersion = new MinecraftVersion(match[2])
const untrackedFiles: UntrackedFilesOption[] = serverMeta.untrackedFiles || []
const modules: Module[] = []
@@ -138,7 +140,8 @@ export class ServerStructure extends BaseModelStructure<Server> {
serverMeta.forge.version,
absoluteServerRoot,
relativeServerRoot,
this.baseUrl
this.baseUrl,
untrackedFiles
)
const forgeModModules = await forgeModStruct.getSpecModel()
@@ -147,16 +150,16 @@ export class ServerStructure extends BaseModelStructure<Server> {
if(serverMeta.liteloader) {
const liteModStruct = new LiteModStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion)
const liteModStruct = new LiteModStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion, untrackedFiles)
const liteModModules = await liteModStruct.getSpecModel()
modules.push(...liteModModules)
}
const libraryStruct = new LibraryStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion)
const libraryStruct = new LibraryStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion, untrackedFiles)
const libraryModules = await libraryStruct.getSpecModel()
modules.push(...libraryModules)
const fileStruct = new MiscFileStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion)
const fileStruct = new MiscFileStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion, untrackedFiles)
const fileModules = await fileStruct.getSpecModel()
modules.push(...fileModules)

6
src/structure/spec_model/module/File.struct.ts
View File

@@ -6,6 +6,7 @@ import { ModuleStructure } from './Module.struct'
import { readdir, stat } from 'fs-extra'
import { join, resolve, sep } from 'path'
import { MinecraftVersion } from '../../../util/MinecraftVersion'
import { UntrackedFilesOption } from '../../../model/nebula/servermeta'
export class MiscFileStructure extends ModuleStructure {
@@ -13,9 +14,10 @@ export class MiscFileStructure extends ModuleStructure {
absoluteRoot: string,
relativeRoot: string,
baseUrl: string,
minecraftVersion: MinecraftVersion
minecraftVersion: MinecraftVersion,
untrackedFiles: UntrackedFilesOption[]
) {
super(absoluteRoot, relativeRoot, 'files', baseUrl, minecraftVersion, Type.File)
super(absoluteRoot, relativeRoot, 'files', baseUrl, minecraftVersion, Type.File, untrackedFiles)
}
public getLoggerName(): string {

6
src/structure/spec_model/module/ForgeMod.struct.ts
View File

@@ -7,6 +7,7 @@ import { MinecraftVersion } from '../../../util/MinecraftVersion'
import { ToggleableModuleStructure } from './ToggleableModule.struct'
import { LibraryType } from '../../../model/claritas/ClaritasLibraryType'
import { ClaritasException } from './Module.struct'
import { UntrackedFilesOption } from '../../../model/nebula/servermeta'
export abstract class BaseForgeModStructure extends ToggleableModuleStructure implements VersionSegmented {
@@ -16,9 +17,10 @@ export abstract class BaseForgeModStructure extends ToggleableModuleStructure im
absoluteRoot: string,
relativeRoot: string,
baseUrl: string,
minecraftVersion: MinecraftVersion
minecraftVersion: MinecraftVersion,
untrackedFiles: UntrackedFilesOption[]
) {
super(absoluteRoot, relativeRoot, 'forgemods', baseUrl, minecraftVersion, Type.ForgeMod)
super(absoluteRoot, relativeRoot, 'forgemods', baseUrl, minecraftVersion, Type.ForgeMod, untrackedFiles)
}
public async getSpecModel(): Promise<Module[]> {

6
src/structure/spec_model/module/Library.struct.ts
View File

@@ -4,6 +4,7 @@ import { Stats } from 'fs-extra'
import { join } from 'path'
import { resolve } from 'url'
import { MinecraftVersion } from '../../../util/MinecraftVersion'
import { UntrackedFilesOption } from '../../../model/nebula/servermeta'
export class LibraryStructure extends ModuleStructure {
@@ -11,9 +12,10 @@ export class LibraryStructure extends ModuleStructure {
absoluteRoot: string,
relativeRoot: string,
baseUrl: string,
minecraftVersion: MinecraftVersion
minecraftVersion: MinecraftVersion,
untrackedFiles: UntrackedFilesOption[]
) {
super(absoluteRoot, relativeRoot, 'libraries', baseUrl, minecraftVersion, Type.Library, (name: string) => {
super(absoluteRoot, relativeRoot, 'libraries', baseUrl, minecraftVersion, Type.Library, untrackedFiles, (name: string) => {
return name.toLowerCase().endsWith(TypeMetadata[this.type].defaultExtension!)
})
}

6
src/structure/spec_model/module/LiteMod.struct.ts
View File

@@ -9,6 +9,7 @@ import { ToggleableModuleStructure } from './ToggleableModule.struct'
import { MinecraftVersion } from '../../../util/MinecraftVersion'
import { LibraryType } from '../../../model/claritas/ClaritasLibraryType'
import { MetadataUtil } from '../../../util/MetadataUtil'
import { UntrackedFilesOption } from '../../../model/nebula/servermeta'
export class LiteModStructure extends ToggleableModuleStructure {
@@ -18,9 +19,10 @@ export class LiteModStructure extends ToggleableModuleStructure {
absoluteRoot: string,
relativeRoot: string,
baseUrl: string,
minecraftVersion: MinecraftVersion
minecraftVersion: MinecraftVersion,
untrackedFiles: UntrackedFilesOption[]
) {
super(absoluteRoot, relativeRoot, 'litemods', baseUrl, minecraftVersion, Type.LiteMod)
super(absoluteRoot, relativeRoot, 'litemods', baseUrl, minecraftVersion, Type.LiteMod, untrackedFiles)
}
public getLoggerName(): string {

41
src/structure/spec_model/module/Module.struct.ts
View File

@@ -1,12 +1,14 @@
import minimatch from 'minimatch'
import { createHash } from 'crypto'
import { lstat, pathExists, readdir, readFile, Stats } from 'fs-extra'
import { Module, Type, TypeMetadata } from 'helios-distribution-types'
import { Artifact, Module, Type, TypeMetadata } from 'helios-distribution-types'
import { resolve } from 'path'
import { BaseModelStructure } from '../BaseModel.struct'
import { LibraryType } from '../../../model/claritas/ClaritasLibraryType'
import { ClaritasResult, ClaritasModuleMetadata } from '../../../model/claritas/ClaritasResult'
import { ClaritasWrapper } from '../../../util/java/ClaritasWrapper'
import { MinecraftVersion } from '../../../util/MinecraftVersion'
import { UntrackedFilesOption } from '../../../model/nebula/servermeta'
export interface ModuleCandidate {
file: string
@@ -24,6 +26,7 @@ export abstract class ModuleStructure extends BaseModelStructure<Module> {
private readonly crudeRegex = /(.+?)-(.+).[jJ][aA][rR]/
protected readonly DEFAULT_VERSION = '0.0.0'
protected untrackedFilePatterns: string[] // List of glob patterns.
protected claritasResult!: ClaritasResult
constructor(
@@ -33,9 +36,11 @@ export abstract class ModuleStructure extends BaseModelStructure<Module> {
baseUrl: string,
protected minecraftVersion: MinecraftVersion,
protected type: Type,
untrackedFiles: UntrackedFilesOption[],
protected filter?: ((name: string, path: string, stats: Stats) => boolean)
) {
super(absoluteRoot, relativeRoot, structRoot, baseUrl)
this.untrackedFilePatterns = this.determineUntrackedFiles(structRoot, untrackedFiles)
}
public async getSpecModel(): Promise<Module[]> {
@@ -87,16 +92,26 @@ export abstract class ModuleStructure extends BaseModelStructure<Module> {
protected async abstract getModulePath(name: string, path: string, stats: Stats): Promise<string | null>
protected async parseModule(file: string, filePath: string, stats: Stats): Promise<Module> {
const artifact: Artifact = {
size: stats.size,
url: await this.getModuleUrl(file, filePath, stats)
}
const relativeToContainer = filePath.substr(this.containerDirectory.length+1)
const untrackedByPattern = this.isFileUntracked(relativeToContainer)
if(!untrackedByPattern) {
const buf = await readFile(filePath)
artifact.MD5 = createHash('md5').update(buf).digest('hex')
} else {
this.logger.debug(`File ${relativeToContainer} is untracked. Matching pattern: ${untrackedByPattern}`)
}
const mdl: Module = {
id: await this.getModuleId(file, filePath),
name: await this.getModuleName(file, filePath),
type: this.type,
artifact: {
size: stats.size,
MD5: createHash('md5').update(buf).digest('hex'),
url: await this.getModuleUrl(file, filePath, stats)
}
artifact
}
const pth = await this.getModulePath(file, filePath, stats)
if (pth) {
@@ -182,4 +197,18 @@ export abstract class ModuleStructure extends BaseModelStructure<Module> {
}
protected determineUntrackedFiles(targetStructRoot: string, untrackedFileOptions?: UntrackedFilesOption[]): string[] {
if(untrackedFileOptions) {
return untrackedFileOptions
.filter(x => x.appliesTo.includes(targetStructRoot))
.reduce((acc, cur) => acc.concat(cur.patterns), [] as string[])
}
return []
}
// Will return the matching pattern, undefined if no match.
protected isFileUntracked(pathRelativeToContainer: string): string | undefined {
return this.untrackedFilePatterns.find(pattern => minimatch(pathRelativeToContainer, pattern))
}
}

4
src/structure/spec_model/module/ToggleableModule.struct.ts
View File

@@ -3,6 +3,7 @@ import { Type, Module } from 'helios-distribution-types'
import { Stats, mkdirs } from 'fs-extra'
import { resolve } from 'path'
import { MinecraftVersion } from '../../../util/MinecraftVersion'
import { UntrackedFilesOption } from '../../../model/nebula/servermeta'
export enum ToggleableNamespace {
@@ -27,9 +28,10 @@ export abstract class ToggleableModuleStructure extends ModuleStructure {
baseUrl: string,
minecraftVersion: MinecraftVersion,
type: Type,
untrackedFiles: UntrackedFilesOption[],
filter?: ((name: string, path: string, stats: Stats) => boolean)
) {
super(absoluteRoot, relativeRoot, structRoot, baseUrl, minecraftVersion, type, filter)
super(absoluteRoot, relativeRoot, structRoot, baseUrl, minecraftVersion, type, untrackedFiles, filter)
}
public async init(): Promise<void> {

6
src/structure/spec_model/module/forgemod/ForgeMod113.struct.ts
View File

@@ -5,6 +5,7 @@ import { VersionUtil } from '../../../../util/versionutil'
import { ModsToml } from '../../../../model/forge/modstoml'
import { BaseForgeModStructure } from '../ForgeMod.struct'
import { MinecraftVersion } from '../../../../util/MinecraftVersion'
import { UntrackedFilesOption } from '../../../../model/nebula/servermeta'
export class ForgeModStructure113 extends BaseForgeModStructure {
@@ -21,9 +22,10 @@ export class ForgeModStructure113 extends BaseForgeModStructure {
absoluteRoot: string,
relativeRoot: string,
baseUrl: string,
minecraftVersion: MinecraftVersion
minecraftVersion: MinecraftVersion,
untrackedFiles: UntrackedFilesOption[]
) {
super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion)
super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, untrackedFiles)
}
public isForVersion(version: MinecraftVersion, libraryVersion: string): boolean {

6
src/structure/spec_model/module/forgemod/ForgeMod17.struct.ts
View File

@@ -6,6 +6,7 @@ import { McModInfoList } from '../../../../model/forge/mcmodinfolist'
import { BaseForgeModStructure } from '../ForgeMod.struct'
import { MinecraftVersion } from '../../../../util/MinecraftVersion'
import { ForgeModType_1_7 } from '../../../../model/claritas/ClaritasResult'
import { UntrackedFilesOption } from '../../../../model/nebula/servermeta'
export class ForgeModStructure17 extends BaseForgeModStructure {
@@ -20,9 +21,10 @@ export class ForgeModStructure17 extends BaseForgeModStructure {
absoluteRoot: string,
relativeRoot: string,
baseUrl: string,
minecraftVersion: MinecraftVersion
minecraftVersion: MinecraftVersion,
untrackedFiles: UntrackedFilesOption[]
) {
super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion)
super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, untrackedFiles)
}
public isForVersion(version: MinecraftVersion, libraryVersion: string): boolean {

6
src/util/VersionSegmentedRegistry.ts
View File

@@ -5,6 +5,7 @@ import { ForgeGradle2Adapter } from '../resolver/forge/adapter/ForgeGradle2.reso
import { ForgeResolver } from '../resolver/forge/forge.resolver'
import { BaseForgeModStructure } from '../structure/spec_model/module/ForgeMod.struct'
import { MinecraftVersion } from './MinecraftVersion'
import { UntrackedFilesOption } from '../model/nebula/servermeta'
export class VersionSegmentedRegistry {
@@ -38,11 +39,12 @@ export class VersionSegmentedRegistry {
forgeVersion: string,
absoluteRoot: string,
relativeRoot: string,
baseUrl: string
baseUrl: string,
untrackedFiles: UntrackedFilesOption[]
): BaseForgeModStructure {
for (const impl of VersionSegmentedRegistry.FORGEMOD_STRUCT_IML) {
if (impl.isForVersion(minecraftVersion, forgeVersion)) {
return new impl(absoluteRoot, relativeRoot, baseUrl, minecraftVersion)
return new impl(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, untrackedFiles)
}
}
throw new Error(`No forge mod structure found for Minecraft ${minecraftVersion}!`)