Merge pull request #2 from anno-mods/devel/fix-buildmod

prepare 0.5.0 with tested buildmod usage

.github/workflows/main.yml

@@ -30,6 +30,8 @@ jobs:
         run: vsce package
       - name: build command line tool
         run: npm run buildmod:pack
+      - name: test 
+        run: npm run test
       - name: upload extension
         uses: actions/upload-artifact@v2
         with:
@@ -47,7 +49,10 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/')
         with:
           draft: true
-          body_path: CHANGELOG.md
+          body: |
+            [Full Changelog](https://github.com/anno-mods/vscode-anno-modding-tools/blob/${{ github.sha }}/CHANGELOG.md)
+
+            Download via [Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=JakobHarder.anno-modding-tools) or in VS Code in the extension tab directly.
           files: |
             anno-modding-tools-${{ steps.package-version.outputs.version }}.vsix
             anno-modding-tools-${{ steps.package-version.outputs.version }}.tgz

.npmignore

@@ -3,10 +3,12 @@ node_modules
 .vscodeignore
 .yarnrc
 .vscode/**
+.vscode-test/**
 **/tsconfig.json
 **/.eslintrc.json
 **/*.map
 webpack.config.js
+out/test/**
 
 # vscode extension related
 dist/**

CHANGELOG.md

@@ -28,16 +28,22 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how
   - Show strong warning when there's no tangent information on the mesh
   - error handling for invalid xml files
   - empty annomod project breaks the extension (tradepatent-fish)
+- Internal
+  - replace all texconv usage to reduce number of tools
+
+## [0.5.0]
 
 ### Added
 
 - Build mod command as script in NPM package for standalone execution
+- First unit tests
 
 ### Changed
 
 - Fixed GUID hover in XPath strings
 - Added Clothes/CLOTH to outlines
 - Internal optimizations
+- Replaced texconv with annotex/bc7enc for better non-GPU build performance
 
 ## [0.4.1]
 

README.md

@@ -66,9 +66,7 @@ A big thanks goes to the external projects I'm using for this extension:
 
 ## Release Notes / Known Issues
 
-### 0.4.0
-
-Assets outline.
+### 0.5.0
 
 See changes and known issues in [CHANGELOG](./CHANGELOG.md)
 

external/download.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 mkdir dl
 
@@ -22,4 +22,7 @@ unzip -o ./dl/filedb.zip -d ./FileDBCompressor
 curl -L https://github.com/jakobharder/rdm4/releases/download/jakob%2F0.6.0-alpha.1%2B3/rdm4-bin-windows-amd64.zip > ./dl/rdm4.zip
 unzip -o ./dl/rdm4.zip
 
+# annotex / bc7enc, MIT
+curl -L https://github.com/jakobharder/annotex/releases/download/jakob%2Fv1.0.0/annotex.exe > ./annotex.exe
+
 rm -r ./dl

package.json

@@ -2,7 +2,7 @@
   "name": "anno-modding-tools",
   "displayName": "Modding Tools for Anno",
   "description": "Modding tools for Anno 1800",
-  "version": "0.5.0-alpha",
+  "version": "0.5.0",
   "publisher": "JakobHarder",
   "repository": {
     "type": "git",
@@ -238,11 +238,11 @@
     "vscode:prepublish": "webpack --mode production",
     "compile": "webpack --mode none",
     "watch": "webpack --mode none --watch",
-    "pretest": "npm run compile && npm run lint",
+    "pretest": "tsc -p ./ && npm run lint",
     "lint": "eslint src --ext ts",
     "test": "node ./out/test/runTest.js",
-    "assets": "tsc ./src/compileAssets.ts --outDir ./out/ && node ./out/compileAssets.js /mnt/c/anno/all-rda/data/config/export/main/asset/assets.xml",
-    "buildmod:compile": "tsc ./src/buildmod.ts --outDir ./out --downlevelIteration",
+    "assets": "tsc -p ./ && node ./out/compileAssets.js /mnt/c/anno/all-rda/data/config/export/main/asset/assets.xml",
+    "buildmod:compile": "tsc -p ./",
     "buildmod:pack": "npm run buildmod:compile && npm pack"
   },
   "devDependencies": {

src/builder/Converter.ts

@@ -17,8 +17,9 @@ export abstract class Converter {
 
   abstract run(files: string[], sourceFolder: string, outFolder: string, options: { 
     cache: string,
+    ci: string,
     modJson: any,
     converterOptions: any,
     dontOverwrite?: boolean
-  }): Promise<void>;
+  }): Promise<boolean>;
 }

src/builder/converter/cf7.ts

@@ -24,8 +24,10 @@ export class Cf7Converter extends Converter {
       }
       catch (exception: any)
       {
-        this._logger.warn(exception);
+        this._logger.error(exception);
+        return false;
       }
     }
+    return true;
   }
 }

src/builder/converter/cfgyaml.ts

@@ -76,8 +76,10 @@ export class CfgYamlConverter extends Converter {
       catch (exception: any)
       {
         this._logger.error(exception.message);
+        return false;
       }
     }
+    return true;
   }
 
   private _runModifications(xml: AnnoXml, modifications: any[]) {

src/builder/converter/gltf.ts

@@ -144,8 +144,11 @@ export class GltfConverter extends Converter {
       catch (exception: any)
       {
         this._logger.error(exception.message);
+        return false;
       }
     }
+
+    return true;
   }
 
   private _replaceImages(gltf: any, fakePngUrl: string) {
@@ -248,15 +251,15 @@ export class GltfConverter extends Converter {
       try {
         ConvertToGLB(gltf, path.join(resourceDirectory, 'fake.gltf'), tempFile);
 
-        const res = child.execFileSync(rdmPath, [
+        const stdout = child.execFileSync(rdmPath, [
           '--gltf-mesh-index', meshIdx.toString(),
           '-g', useSkeleton ? 'P4h_N4b_G4b_B4b_T2h_I4b' : 'P4h_N4b_G4b_B4b_T2h',
           '-n', '-o', path.dirname(targetFile),
           '-i', tempFile,
           ...(useSkeleton ? [ '--skeleton' ] : []),
           ...(useAnimation ? [ '--animation'] : []),
           '--force' // overwrite existing files
-        ]);
+        ], { stdio : 'pipe' });
         resolve(true);
       } catch (exception) {
         reject(exception);

src/builder/converter/modinfo.ts

@@ -54,8 +54,10 @@ export class ModinfoConverter extends Converter {
     }
     catch (exception: any)
     {
-      this._logger.log(exception.message);
+      this._logger.error(exception.message);
+      return false;
     }
+    return true;
   }
 
   private _readMarkdownWithoutImages(sourceFolder: string, filePath: string) {

src/builder/converter/rdpxml.ts

@@ -27,8 +27,10 @@ export class RdpxmlConverter extends Converter {
       }
       catch (exception: any)
       {
-        this._logger.warn(exception);
+        this._logger.error(exception);
+        return false;
       }
     }
+    return true;
   }
 }

src/builder/converter/static.ts

@@ -22,7 +22,9 @@ export class StaticConverter extends Converter {
       catch (exception: any)
       {
         this._logger.error(exception.message);
+        return false;
       }
     }
+    return true;
   }
 }

src/builder/converter/texture.ts

@@ -12,6 +12,7 @@ export class TextureConverter extends Converter {
 
   public async run(files: string[], sourceFolder: string, outFolder: string, options: {
     cache: string,
+    ci: string,
     converterOptions: any
   }) {
     for (const file of files) {
@@ -39,49 +40,21 @@ export class TextureConverter extends Converter {
           }
         }
 
-        // save first target
-        const tmpFilePath = path.join(options.cache, dirname);
-        dds.convertToTexture(sourceFile, tmpFilePath, sourceFile.endsWith(maskEnding) ? dds.TextureFormat.bc1Unorm : dds.TextureFormat.bc7Unorm);
-        // unfortunately, texconv doesn't allow to change the output file name
-        fs.renameSync(path.join(tmpFilePath, basename + '.dds'), lodFilePaths[0]);
-        this._logger.log(`  <= ${lodLevels ? `LOD ${0}: ` : ''}${path.relative(path.dirname(file), path.relative(outFolder, lodFilePaths[0]))}`);
-
-        // generate lods by reading out previous .dds mipmaps
-        if (lodFilePaths.length > 1) {
-          this._extractLodsFromDds(lodFilePaths.shift() as string, lodFilePaths, path.join(outFolder, dirname));
+        const targetFolder = path.dirname(lodFilePaths[0]);
+        const textures = dds.convertToTexture(sourceFile, targetFolder, dds.TextureFormat.unknown, lodLevels);
+        if (!textures) {
+          return false;
+        }
+        for (var [index, texture] of textures.entries()) {
+          this._logger.log(`  <= ${lodLevels ? `LOD ${index}: ` : ''}${path.relative(path.dirname(file), path.relative(outFolder, path.join(targetFolder, texture)))}`);
         }
       }
       catch (exception: any)
       {
         this._logger.error(exception.message);
+        return false;
       }
     }
-  }
-
-  private _extractLodsFromDds(source: string, targets: string[], outFolder: string) {
-    const texture = dds.Texture.fromFile(source);
-    if (!texture) {
-      return;
-    }
-
-    const mipmaps = texture.images;
-    let width = texture.width;
-    let height = texture.height;
-    for (let level = 0; level < targets.length && width > 1 && height > 1; level++) {
-      // go one mipmap down
-      mipmaps.shift();
-      if (mipmaps.length === 0) {
-        this._logger.warn(`     LOD ${level + 1}: Skip LOD for ${width}x${height} because of missing source mipmap.`);
-        break; // no more mipmaps available
-      }
-      width = Math.floor((width + 1) / 2);
-      height = Math.floor((height + 1) / 2);
-      // dump
-      fs.writeFileSync(targets[level], Buffer.concat([ 
-        texture.getModifiedHeader(width, height, texture.mipmaps - level - 1), 
-        ...texture.images
-      ]));
-      this._logger.log(`  <= LOD ${level + 1}: ${path.relative(outFolder, targets[level])}`);
-    }
+    return true;
   }
 }

src/builder/index.ts

@@ -11,7 +11,11 @@ import { ModinfoConverter } from './converter/modinfo';
 import { RdpxmlConverter } from './converter/rdpxml';
 import { CfgYamlConverter } from './converter/cfgyaml';
 
+import * as rdp from '../other/rdp';
+import * as dds from '../other/dds';
+
 import * as xmltest from '../other/xmltest';
+import * as utils from '../other/utils';
 
 export class ModBuilder {
   _converters: { [index: string]: Converter } = {};
@@ -20,6 +24,9 @@ export class ModBuilder {
   _variableAnnoMods;
 
   public constructor(logger: ILogger, asAbsolutePath: (relative: string) => string, variableAnnoMods: string) {
+    rdp.init(asAbsolutePath('./external/'));
+    dds.init(asAbsolutePath('./external/'));
+
     this._logger = logger;
     this._asAbsolutePath = asAbsolutePath;
     this._variableAnnoMods = variableAnnoMods;
@@ -46,7 +53,10 @@ export class ModBuilder {
       sourceFolder += '/';
     }
     const outFolder = this._getOutFolder(filePath, modJson);
-    const cacheFolder = path.join(path.dirname(filePath), '.modcache');
+    const cache = path.join(path.dirname(filePath), '.modcache');
+    const ci = path.join(path.dirname(filePath), '.vanilla');
+    // utils.ensureDir(ci);
+    // fs.writeFileSync(path.join(ci, 'readme.md'), `This folder contains unmodified assets from the original game to allow CI like GitHub actions to build the mod without RDA data.`);
 
     this._logger.log('Target folder: ' + outFolder);
 
@@ -63,27 +73,35 @@ export class ModBuilder {
       const converter = this._converters[entry.action];
       if (converter) {
         this._logger.log(`${entry.action}` + (entry.pattern?`: ${entry.pattern}`:''));
-        await converter.run(allFiles, sourceFolder, outFolder, {
-          cache: cacheFolder,
+        const result = await converter.run(allFiles, sourceFolder, outFolder, {
+          cache,
+          ci,
           modJson,
           converterOptions: entry
         });
+        if (!result) {
+          return false;
+        }
       }
       else {
-        this._logger.log('Error: no converter with name: ' + entry.action);
+        this._logger.error('Error: no converter with name: ' + entry.action);
+        return false;
       }
     }
 
     const testFolder = path.join(sourceFolder, 'tests');
     if (fs.existsSync(sourceFolder)) {
       this._logger.log(`Run tests from ${testFolder}`);
-      xmltest.test(testFolder, path.join(sourceFolder, 'data/config/export/main/asset/assets.xml'), this._asAbsolutePath, cacheFolder);
+      if (!xmltest.test(testFolder, path.join(sourceFolder, 'data/config/export/main/asset/assets.xml'), this._asAbsolutePath, cache)) {
+        return false;
+      }
     }
     else {
       this._logger.log(`No test folder available: ${testFolder}`);
     }
 
     this._logger.log(`${this._getModName(filePath, modJson)} done`);
+    return true;
   }
 
   private _getOutFolder(filePath: string, modJson: any) {