Add support for untracked files.

Untracked file glob patterns are stored in the servermeta.json. See the README for detailed information.

README.md

@@ -240,10 +240,51 @@ Represents the additional metadata on the server object (for a YOUR_SERVER). Sam
   },
   "forge": {
     "version": "14.23.5.2854"
-  }
+  },
+  "untrackedFiles": []
 }
 ```
 
+Untracked files is optional. MD5 hashes will not be generated for files matching the provided glob patterns.
+
+```json
+{
+  "untrackedFiles": [
+    {
+      "appliesTo": ["files"],
+      "patterns": [
+        "config/*.cfg",
+        "config/**/*.yml"
+      ]
+    }
+  ]
+}
+```
+
+In the above example, all files of type `cfg` in the config directory will be untracked. Additionally, all files of type `yml` in the config directory and its subdirectories will be untracked. You can tweak these patterns to fit your needs, this is purely an example. The patterns will only be applied to the folders specified in `appliesTo`. As an example, valid values include `files`, `forgemods`, `libraries`, etc.
+
+```json
+{
+  "untrackedFiles": [
+    {
+      "appliesTo": ["files"],
+      "patterns": [
+        "config/*.cfg",
+        "config/**/*.yml"
+      ]
+    },
+    {
+      "appliesTo": ["forgemods", "litemods"],
+      "patterns": [
+        "optionalon/*.jar"
+      ]
+    }
+  ]
+}
+```
+
+Another example where all `optionalon` forgemods and litemods are untracked. **Untracking mods is NOT recommended. This is an example ONLY.**
+
 
 [dotenvnpm]: https://www.npmjs.com/package/dotenv
 [distro.md]: https://github.com/dscalzi/HeliosLauncher/blob/master/docs/distro.md

package-lock.json

@@ -186,6 +186,12 @@
         "@types/node": "*"
       }
     },
+    "@types/minimatch": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
+      "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==",
+      "dev": true
+    },
     "@types/node": {
       "version": "12.12.58",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.58.tgz",
@@ -381,14 +387,12 @@
     "balanced-match": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
-      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
-      "dev": true
     },
     "brace-expansion": {
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
       "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
-      "dev": true,
       "requires": {
         "balanced-match": "^1.0.0",
         "concat-map": "0.0.1"
@@ -536,8 +540,7 @@
     "concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
-      "dev": true
     },
     "core-util-is": {
       "version": "1.0.2",
@@ -1028,9 +1031,9 @@
       "dev": true
     },
     "helios-distribution-types": {
-      "version": "1.0.0-pre.1",
+      "version": "1.0.0-rc.1",
-      "resolved": "https://registry.npmjs.org/helios-distribution-types/-/helios-distribution-types-1.0.0-pre.1.tgz",
+      "resolved": "https://registry.npmjs.org/helios-distribution-types/-/helios-distribution-types-1.0.0-rc.1.tgz",
-      "integrity": "sha512-rqmuLoiyZTLGH0rlklRpjTvVbBTtP/NOyQqCvBPaLwpU2xUX/Vxzmt3RE8k+OgN3BAUEVJ+4jhSLgCsCXJfx1g=="
+      "integrity": "sha512-b3GNQoTDexFZ7CvW5F2vJrKbWpQZKof1uWOWT9vcvrPE91WLhzZf3UBdBOzIlSHpXiu58dZxh7eLZw+HUE7YLA=="
     },
     "http-cache-semantics": {
       "version": "4.1.0",
@@ -1250,7 +1253,6 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
       "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
-      "dev": true,
       "requires": {
         "brace-expansion": "^1.1.7"
       }

package.json

@@ -27,6 +27,7 @@
   "homepage": "https://github.com/dscalzi/Nebula#readme",
   "devDependencies": {
     "@types/fs-extra": "^9.0.1",
+    "@types/minimatch": "^3.0.3",
     "@types/node": "^12.12.58",
     "@types/triple-beam": "^1.3.2",
     "@types/yargs": "^15.0.5",
@@ -40,7 +41,8 @@
     "dotenv": "^8.2.0",
     "fs-extra": "^9.0.1",
     "got": "^11.6.2",
-    "helios-distribution-types": "^1.0.0-pre.1",
+    "helios-distribution-types": "^1.0.0-rc.1",
+    "minimatch": "^3.0.4",
     "moment": "^2.27.0",
     "node-stream-zip": "^1.11.3",
     "toml": "^3.0.0",

src/model/nebula/servermeta.ts

@@ -1,5 +1,17 @@
 import { Server } from 'helios-distribution-types'
 
+export interface UntrackedFilesOption {
+    /**
+     * The subdirectory this applies to. Ex.
+     * [ 'files', 'forgemods' ]
+     */
+    appliesTo: string[]
+    /**
+     * Glob patterns to match against the file.
+     */
+    patterns: string[]
+}
+
 export interface ServerMetaOptions {
     forgeVersion?: string
     liteloaderVersion?: string
@@ -37,6 +49,9 @@ export function getDefaultServerMeta(id: string, version: string, options?: Serv
         }
     }
 
+    // Add empty untracked files.
+    servMeta.untrackedFiles = []
+
     return servMeta
 }
 
@@ -60,4 +75,6 @@ export interface ServerMeta {
         version: string
     }
 
+    untrackedFiles?: UntrackedFilesOption[]
+
 }

src/structure/spec_model/Server.struct.ts

@@ -3,7 +3,7 @@ import { Server, Module } from 'helios-distribution-types'
 import { dirname, join, resolve as resolvePath } from 'path'
 import { resolve as resolveUrl } from 'url'
 import { VersionSegmentedRegistry } from '../../util/VersionSegmentedRegistry'
-import { ServerMeta, getDefaultServerMeta, ServerMetaOptions } from '../../model/nebula/servermeta'
+import { ServerMeta, getDefaultServerMeta, ServerMetaOptions, UntrackedFilesOption } from '../../model/nebula/servermeta'
 import { BaseModelStructure } from './BaseModel.struct'
 import { MiscFileStructure } from './module/File.struct'
 import { LiteModStructure } from './module/LiteMod.struct'
@@ -60,14 +60,15 @@ export class ServerStructure extends BaseModelStructure<Server> {
                 options.forgeVersion,
                 absoluteServerRoot,
                 relativeServerRoot,
-                this.baseUrl
+                this.baseUrl,
+                []
             )
             await fms.init()
             serverMetaOpts.forgeVersion = options.forgeVersion
         }
 
         if (options.liteloaderVersion != null) {
-            const lms = new LiteModStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion)
+            const lms = new LiteModStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion, [])
             await lms.init()
             serverMetaOpts.liteloaderVersion = options.liteloaderVersion
         }
@@ -75,10 +76,10 @@ export class ServerStructure extends BaseModelStructure<Server> {
         const serverMeta: ServerMeta = getDefaultServerMeta(id, minecraftVersion.toString(), serverMetaOpts)
         await writeFile(resolvePath(absoluteServerRoot, this.SERVER_META_FILE), JSON.stringify(serverMeta, null, 2))
 
-        const libS = new LibraryStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion)
+        const libS = new LibraryStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion, [])
         await libS.init()
 
-        const mfs = new MiscFileStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion)
+        const mfs = new MiscFileStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion, [])
         await mfs.init()
 
     }
@@ -117,6 +118,7 @@ export class ServerStructure extends BaseModelStructure<Server> {
                 // Read server meta
                 const serverMeta: ServerMeta = JSON.parse(await readFile(resolvePath(absoluteServerRoot, this.SERVER_META_FILE), 'utf-8'))
                 const minecraftVersion = new MinecraftVersion(match[2])
+                const untrackedFiles: UntrackedFilesOption[] = serverMeta.untrackedFiles || []
 
                 const modules: Module[] = []
 
@@ -138,7 +140,8 @@ export class ServerStructure extends BaseModelStructure<Server> {
                         serverMeta.forge.version,
                         absoluteServerRoot,
                         relativeServerRoot,
-                        this.baseUrl
+                        this.baseUrl,
+                        untrackedFiles
                     )
 
                     const forgeModModules = await forgeModStruct.getSpecModel()
@@ -147,16 +150,16 @@ export class ServerStructure extends BaseModelStructure<Server> {
 
                 
                 if(serverMeta.liteloader) {
-                    const liteModStruct = new LiteModStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion)
+                    const liteModStruct = new LiteModStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion, untrackedFiles)
                     const liteModModules = await liteModStruct.getSpecModel()
                     modules.push(...liteModModules)
                 }
 
-                const libraryStruct = new LibraryStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion)
+                const libraryStruct = new LibraryStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion, untrackedFiles)
                 const libraryModules = await libraryStruct.getSpecModel()
                 modules.push(...libraryModules)
 
-                const fileStruct = new MiscFileStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion)
+                const fileStruct = new MiscFileStructure(absoluteServerRoot, relativeServerRoot, this.baseUrl, minecraftVersion, untrackedFiles)
                 const fileModules = await fileStruct.getSpecModel()
                 modules.push(...fileModules)
 

src/structure/spec_model/module/File.struct.ts

@@ -6,6 +6,7 @@ import { ModuleStructure } from './Module.struct'
 import { readdir, stat } from 'fs-extra'
 import { join, resolve, sep } from 'path'
 import { MinecraftVersion } from '../../../util/MinecraftVersion'
+import { UntrackedFilesOption } from '../../../model/nebula/servermeta'
 
 export class MiscFileStructure extends ModuleStructure {
 
@@ -13,9 +14,10 @@ export class MiscFileStructure extends ModuleStructure {
         absoluteRoot: string,
         relativeRoot: string,
         baseUrl: string,
-        minecraftVersion: MinecraftVersion
+        minecraftVersion: MinecraftVersion,
+        untrackedFiles: UntrackedFilesOption[]
     ) {
-        super(absoluteRoot, relativeRoot, 'files', baseUrl, minecraftVersion, Type.File)
+        super(absoluteRoot, relativeRoot, 'files', baseUrl, minecraftVersion, Type.File, untrackedFiles)
     }
 
     public getLoggerName(): string {

src/structure/spec_model/module/ForgeMod.struct.ts

@@ -7,6 +7,7 @@ import { MinecraftVersion } from '../../../util/MinecraftVersion'
 import { ToggleableModuleStructure } from './ToggleableModule.struct'
 import { LibraryType } from '../../../model/claritas/ClaritasLibraryType'
 import { ClaritasException } from './Module.struct'
+import { UntrackedFilesOption } from '../../../model/nebula/servermeta'
 
 export abstract class BaseForgeModStructure extends ToggleableModuleStructure implements VersionSegmented {
 
@@ -16,9 +17,10 @@ export abstract class BaseForgeModStructure extends ToggleableModuleStructure im
         absoluteRoot: string,
         relativeRoot: string,
         baseUrl: string,
-        minecraftVersion: MinecraftVersion
+        minecraftVersion: MinecraftVersion,
+        untrackedFiles: UntrackedFilesOption[]
     ) {
-        super(absoluteRoot, relativeRoot, 'forgemods', baseUrl, minecraftVersion, Type.ForgeMod)
+        super(absoluteRoot, relativeRoot, 'forgemods', baseUrl, minecraftVersion, Type.ForgeMod, untrackedFiles)
     }
 
     public async getSpecModel(): Promise<Module[]> {

src/structure/spec_model/module/Library.struct.ts

@@ -4,6 +4,7 @@ import { Stats } from 'fs-extra'
 import { join } from 'path'
 import { resolve } from 'url'
 import { MinecraftVersion } from '../../../util/MinecraftVersion'
+import { UntrackedFilesOption } from '../../../model/nebula/servermeta'
 
 export class LibraryStructure extends ModuleStructure {
 
@@ -11,9 +12,10 @@ export class LibraryStructure extends ModuleStructure {
         absoluteRoot: string,
         relativeRoot: string,
         baseUrl: string,
-        minecraftVersion: MinecraftVersion
+        minecraftVersion: MinecraftVersion,
+        untrackedFiles: UntrackedFilesOption[]
     ) {
-        super(absoluteRoot, relativeRoot, 'libraries', baseUrl, minecraftVersion, Type.Library, (name: string) => {
+        super(absoluteRoot, relativeRoot, 'libraries', baseUrl, minecraftVersion, Type.Library, untrackedFiles, (name: string) => {
             return name.toLowerCase().endsWith(TypeMetadata[this.type].defaultExtension!)
         })
     }

src/structure/spec_model/module/LiteMod.struct.ts

@@ -9,6 +9,7 @@ import { ToggleableModuleStructure } from './ToggleableModule.struct'
 import { MinecraftVersion } from '../../../util/MinecraftVersion'
 import { LibraryType } from '../../../model/claritas/ClaritasLibraryType'
 import { MetadataUtil } from '../../../util/MetadataUtil'
+import { UntrackedFilesOption } from '../../../model/nebula/servermeta'
 
 export class LiteModStructure extends ToggleableModuleStructure {
 
@@ -18,9 +19,10 @@ export class LiteModStructure extends ToggleableModuleStructure {
         absoluteRoot: string,
         relativeRoot: string,
         baseUrl: string,
-        minecraftVersion: MinecraftVersion
+        minecraftVersion: MinecraftVersion,
+        untrackedFiles: UntrackedFilesOption[]
     ) {
-        super(absoluteRoot, relativeRoot, 'litemods', baseUrl, minecraftVersion, Type.LiteMod)
+        super(absoluteRoot, relativeRoot, 'litemods', baseUrl, minecraftVersion, Type.LiteMod, untrackedFiles)
     }
 
     public getLoggerName(): string {

src/structure/spec_model/module/Module.struct.ts

@@ -1,12 +1,14 @@
+import minimatch from 'minimatch'
 import { createHash } from 'crypto'
 import { lstat, pathExists, readdir, readFile, Stats } from 'fs-extra'
-import { Module, Type, TypeMetadata } from 'helios-distribution-types'
+import { Artifact, Module, Type, TypeMetadata } from 'helios-distribution-types'
 import { resolve } from 'path'
 import { BaseModelStructure } from '../BaseModel.struct'
 import { LibraryType } from '../../../model/claritas/ClaritasLibraryType'
 import { ClaritasResult, ClaritasModuleMetadata } from '../../../model/claritas/ClaritasResult'
 import { ClaritasWrapper } from '../../../util/java/ClaritasWrapper'
 import { MinecraftVersion } from '../../../util/MinecraftVersion'
+import { UntrackedFilesOption } from '../../../model/nebula/servermeta'
 
 export interface ModuleCandidate {
     file: string
@@ -24,6 +26,7 @@ export abstract class ModuleStructure extends BaseModelStructure<Module> {
     private readonly crudeRegex = /(.+?)-(.+).[jJ][aA][rR]/
     protected readonly DEFAULT_VERSION = '0.0.0'
 
+    protected untrackedFilePatterns: string[]          // List of glob patterns. 
     protected claritasResult!: ClaritasResult
 
     constructor(
@@ -33,9 +36,11 @@ export abstract class ModuleStructure extends BaseModelStructure<Module> {
         baseUrl: string,
         protected minecraftVersion: MinecraftVersion,
         protected type: Type,
+        untrackedFiles: UntrackedFilesOption[],
         protected filter?: ((name: string, path: string, stats: Stats) => boolean)
     ) {
         super(absoluteRoot, relativeRoot, structRoot, baseUrl)
+        this.untrackedFilePatterns = this.determineUntrackedFiles(structRoot, untrackedFiles)
     }
 
     public async getSpecModel(): Promise<Module[]> {
@@ -87,16 +92,26 @@ export abstract class ModuleStructure extends BaseModelStructure<Module> {
     protected async abstract getModulePath(name: string, path: string, stats: Stats): Promise<string | null>
 
     protected async parseModule(file: string, filePath: string, stats: Stats): Promise<Module> {
-        const buf = await readFile(filePath)
+
+        const artifact: Artifact = {
+            size: stats.size,
+            url: await this.getModuleUrl(file, filePath, stats)
+        }
+        
+        const relativeToContainer = filePath.substr(this.containerDirectory.length+1)
+        const untrackedByPattern = this.isFileUntracked(relativeToContainer)
+        if(!untrackedByPattern) {
+            const buf = await readFile(filePath)
+            artifact.MD5 = createHash('md5').update(buf).digest('hex')
+        } else {
+            this.logger.debug(`File ${relativeToContainer} is untracked. Matching pattern: ${untrackedByPattern}`)
+        }
+        
         const mdl: Module = {
             id: await this.getModuleId(file, filePath),
             name: await this.getModuleName(file, filePath),
             type: this.type,
-            artifact: {
+            artifact
-                size: stats.size,
-                MD5: createHash('md5').update(buf).digest('hex'),
-                url: await this.getModuleUrl(file, filePath, stats)
-            }
         }
         const pth = await this.getModulePath(file, filePath, stats)
         if (pth) {
@@ -182,4 +197,18 @@ export abstract class ModuleStructure extends BaseModelStructure<Module> {
 
     }
 
+    protected determineUntrackedFiles(targetStructRoot: string, untrackedFileOptions?: UntrackedFilesOption[]): string[] {
+        if(untrackedFileOptions) {
+            return untrackedFileOptions
+                .filter(x => x.appliesTo.includes(targetStructRoot))
+                .reduce((acc, cur) => acc.concat(cur.patterns), [] as string[])
+        }
+        return []
+    }
+
+    // Will return the matching pattern, undefined if no match.
+    protected isFileUntracked(pathRelativeToContainer: string): string | undefined {
+        return this.untrackedFilePatterns.find(pattern => minimatch(pathRelativeToContainer, pattern))
+    }
+
 }

src/structure/spec_model/module/ToggleableModule.struct.ts

@@ -3,6 +3,7 @@ import { Type, Module } from 'helios-distribution-types'
 import { Stats, mkdirs } from 'fs-extra'
 import { resolve } from 'path'
 import { MinecraftVersion } from '../../../util/MinecraftVersion'
+import { UntrackedFilesOption } from '../../../model/nebula/servermeta'
 
 export enum ToggleableNamespace {
 
@@ -27,9 +28,10 @@ export abstract class ToggleableModuleStructure extends ModuleStructure {
         baseUrl: string,
         minecraftVersion: MinecraftVersion,
         type: Type,
+        untrackedFiles: UntrackedFilesOption[],
         filter?: ((name: string, path: string, stats: Stats) => boolean)
     ) {
-        super(absoluteRoot, relativeRoot, structRoot, baseUrl, minecraftVersion, type, filter)
+        super(absoluteRoot, relativeRoot, structRoot, baseUrl, minecraftVersion, type, untrackedFiles, filter)
     }
 
     public async init(): Promise<void> {

src/structure/spec_model/module/forgemod/ForgeMod113.struct.ts

@@ -5,6 +5,7 @@ import { VersionUtil } from '../../../../util/versionutil'
 import { ModsToml } from '../../../../model/forge/modstoml'
 import { BaseForgeModStructure } from '../ForgeMod.struct'
 import { MinecraftVersion } from '../../../../util/MinecraftVersion'
+import { UntrackedFilesOption } from '../../../../model/nebula/servermeta'
 
 export class ForgeModStructure113 extends BaseForgeModStructure {
 
@@ -21,9 +22,10 @@ export class ForgeModStructure113 extends BaseForgeModStructure {
         absoluteRoot: string,
         relativeRoot: string,
         baseUrl: string,
-        minecraftVersion: MinecraftVersion
+        minecraftVersion: MinecraftVersion,
+        untrackedFiles: UntrackedFilesOption[]
     ) {
-        super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion)
+        super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, untrackedFiles)
     }
 
     public isForVersion(version: MinecraftVersion, libraryVersion: string): boolean {

src/structure/spec_model/module/forgemod/ForgeMod17.struct.ts

@@ -6,6 +6,7 @@ import { McModInfoList } from '../../../../model/forge/mcmodinfolist'
 import { BaseForgeModStructure } from '../ForgeMod.struct'
 import { MinecraftVersion } from '../../../../util/MinecraftVersion'
 import { ForgeModType_1_7 } from '../../../../model/claritas/ClaritasResult'
+import { UntrackedFilesOption } from '../../../../model/nebula/servermeta'
 
 export class ForgeModStructure17 extends BaseForgeModStructure {
 
@@ -20,9 +21,10 @@ export class ForgeModStructure17 extends BaseForgeModStructure {
         absoluteRoot: string,
         relativeRoot: string,
         baseUrl: string,
-        minecraftVersion: MinecraftVersion
+        minecraftVersion: MinecraftVersion,
+        untrackedFiles: UntrackedFilesOption[]
     ) {
-        super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion)
+        super(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, untrackedFiles)
     }
 
     public isForVersion(version: MinecraftVersion, libraryVersion: string): boolean {

src/util/VersionSegmentedRegistry.ts

@@ -5,6 +5,7 @@ import { ForgeGradle2Adapter } from '../resolver/forge/adapter/ForgeGradle2.reso
 import { ForgeResolver } from '../resolver/forge/forge.resolver'
 import { BaseForgeModStructure } from '../structure/spec_model/module/ForgeMod.struct'
 import { MinecraftVersion } from './MinecraftVersion'
+import { UntrackedFilesOption } from '../model/nebula/servermeta'
 
 export class VersionSegmentedRegistry {
 
@@ -38,11 +39,12 @@ export class VersionSegmentedRegistry {
         forgeVersion: string,
         absoluteRoot: string,
         relativeRoot: string,
-        baseUrl: string
+        baseUrl: string,
+        untrackedFiles: UntrackedFilesOption[]
     ): BaseForgeModStructure {
         for (const impl of VersionSegmentedRegistry.FORGEMOD_STRUCT_IML) {
             if (impl.isForVersion(minecraftVersion, forgeVersion)) {
-                return new impl(absoluteRoot, relativeRoot, baseUrl, minecraftVersion)
+                return new impl(absoluteRoot, relativeRoot, baseUrl, minecraftVersion, untrackedFiles)
             }
         }
         throw new Error(`No forge mod structure found for Minecraft ${minecraftVersion}!`)