Merge pull request microsoft#1812 from iclanton/ianc/normalize-newlines

[localization-plugin] Add support for normalization of newlines in RESX files.

Author: iclanton

build-tests/localization-plugin-test-02/webpack.config.js

@@ -66,7 +66,8 @@ module.exports = function(env) {
           passthroughLocale: {
             usePassthroughLocale: true,
             passthroughLocaleName: 'default'
-          }
+          },
+          normalizeResxNewlines: 'crlf'
         },
         typingsOptions: {
           generatedTsFolder: path.resolve(__dirname, 'temp', 'loc-json-ts'),

build-tests/localization-plugin-test-03/webpack.config.js

@@ -104,7 +104,8 @@ module.exports = function(env) {
               append: '##--!!]',
               prepend: '[!!--##'
             }
-          }
+          },
+          normalizeResxNewlines: 'lf'
         },
         typingsOptions: {
           generatedTsFolder: path.resolve(__dirname, 'temp', 'loc-json-ts'),

common/changes/@rushstack/localization-plugin/ianc-normalize-newlines_2020-04-05-06-55.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/localization-plugin",
+      "comment": "Add support for normalization of newlines in RESX files.",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@rushstack/localization-plugin",
+  "email": "[email protected]"
+}

common/reviews/api/localization-plugin.api.md

@@ -4,6 +4,7 @@
 
 ```ts
 
+import { NewlineKind } from '@rushstack/node-core-library';
 import { StringValuesTypingsGenerator } from '@rushstack/typings-generator';
 import { Terminal } from '@rushstack/node-core-library';
 import * as Webpack from 'webpack';
@@ -80,6 +81,7 @@ export interface ILocalizationStatsOptions {
 // @public (undocumented)
 export interface ILocalizedData {
     defaultLocale: IDefaultLocaleOptions;
+    normalizeResxNewlines?: 'lf' | 'crlf';
     passthroughLocale?: IPassthroughLocaleOptions;
     pseudolocales?: IPseudolocalesOptions;
     resolveMissingTranslatedStrings?: (locales: string[], filePath: string) => IResolvedMissingTranslations;
@@ -115,6 +117,8 @@ export interface _IParseLocFileOptions {
     // (undocumented)
     filePath: string;
     // (undocumented)
+    resxNewlineNormalization: NewlineKind | undefined;
+    // (undocumented)
     terminal: Terminal;
 }
 
@@ -178,6 +182,8 @@ export interface ITypingsGeneratorOptions {
     // (undocumented)
     generatedTsFolder: string;
     // (undocumented)
+    resxNewlineNormalization?: NewlineKind | undefined;
+    // (undocumented)
     srcFolder: string;
     // (undocumented)
     terminal?: Terminal;

webpack/localization-plugin/README.md

@@ -178,6 +178,14 @@ This option allows pseudolocales to be generated from the strings in the default
 an option with pseudolocales as keys and options for the
 [pseudolocale package](https://www.npmjs.com/package/pseudolocale) as values.
 
+#### `localizedData.normalizeResxNewlines = 'crlf' | 'lf'`
+
+This option allows normalization of newlines in RESX files. RESX files are XML, so newlines can be
+specified by including a newline in the `<value>` element. For files stored on source control systems,
+clones on Windows can end up with CRLF newlines and clones on 'nix operating systems can end up with LF
+newlines. This option can be used to help make compilations run on different platforms produce the same
+result.
+
 ### `filesToIgnore = [ ]`
 
 This option is used to specify `.resx` and `.loc.json` files that should not be processed by this plugin.

webpack/localization-plugin/src/LocFileTypingsGenerator.ts

@@ -5,7 +5,10 @@ import {
   StringValuesTypingsGenerator,
   IStringValueTyping
 } from '@rushstack/typings-generator';
-import { Terminal } from '@rushstack/node-core-library';
+import {
+  Terminal,
+  NewlineKind
+} from '@rushstack/node-core-library';
 
 import { ILocalizationFile } from './interfaces';
 import { LocFileParser } from './utilities/LocFileParser';
@@ -19,6 +22,7 @@ export interface ITypingsGeneratorOptions {
   terminal?: Terminal;
   exportAsDefault?: boolean;
   filesToIgnore?: string[];
+  resxNewlineNormalization?: NewlineKind | undefined;
 }
 
 /**
@@ -35,7 +39,8 @@ export class LocFileTypingsGenerator extends StringValuesTypingsGenerator {
         const locFileData: ILocalizationFile = LocFileParser.parseLocFile({
           filePath: filePath,
           content: fileContents,
-          terminal: this._options.terminal!
+          terminal: this._options.terminal!,
+          resxNewlineNormalization: options.resxNewlineNormalization
         });
 
         const typings: IStringValueTyping[] = [];

webpack/localization-plugin/src/LocalizationPlugin.ts

@@ -4,7 +4,8 @@
 import {
   JsonFile,
   FileSystem,
-  Terminal
+  Terminal,
+  NewlineKind
 } from '@rushstack/node-core-library';
 import * as Webpack from 'webpack';
 import * as path from 'path';
@@ -111,6 +112,7 @@ export class LocalizationPlugin implements Webpack.Plugin {
   private _noStringsLocaleName: string;
   private _fillMissingTranslationStrings: boolean;
   private _pseudolocalizers: Map<string, (str: string) => string> = new Map<string, (str: string) => string>();
+  private _resxNewlineNormalization: NewlineKind | undefined;
 
   /**
    * The outermost map's keys are the locale names.
@@ -173,7 +175,8 @@ export class LocalizationPlugin implements Webpack.Plugin {
       pluginInstance: this,
       configuration: compiler.options,
       filesToIgnore: this._filesToIgnore,
-      localeNameOrPlaceholder: Constants.LOCALE_NAME_PLACEHOLDER
+      localeNameOrPlaceholder: Constants.LOCALE_NAME_PLACEHOLDER,
+      resxNewlineNormalization: this._resxNewlineNormalization
     };
 
     if (errors.length > 0 || warnings.length > 0) {
@@ -432,7 +435,8 @@ export class LocalizationPlugin implements Webpack.Plugin {
         const localizationFile: ILocalizationFile = LocFileParser.parseLocFile({
           filePath: localizedData,
           content: FileSystem.readFile(localizedData),
-          terminal: terminal
+          terminal: terminal,
+          resxNewlineNormalization: this._resxNewlineNormalization
         });
 
         return this._convertLocalizationFileToLocData(localizationFile);
@@ -690,6 +694,30 @@ export class LocalizationPlugin implements Webpack.Plugin {
         }
       }
       // END options.localizedData.pseudoLocales
+
+      // START options.localizedData.normalizeResxNewlines
+      if (this._options.localizedData.normalizeResxNewlines) {
+        switch (this._options.localizedData.normalizeResxNewlines) {
+          case 'crlf': {
+            this._resxNewlineNormalization = NewlineKind.CrLf;
+            break;
+          }
+
+          case 'lf': {
+            this._resxNewlineNormalization = NewlineKind.Lf;
+            break;
+          }
+
+          default: {
+            errors.push(new Error(
+              `Unexpected value "${this._options.localizedData.normalizeResxNewlines}" for option ` +
+              '"localizedData.normalizeResxNewlines"'
+            ));
+            break;
+          }
+        }
+      }
+      // END options.localizedData.normalizeResxNewlines
     } else if (!isWebpackDevServer) {
       throw new Error('Localized data must be provided unless webpack dev server is running.');
     }

webpack/localization-plugin/src/WebpackConfigurationUpdater.ts

@@ -4,6 +4,7 @@
 import * as path from 'path';
 import * as Webpack from 'webpack';
 import * as SetPublicPathPluginPackageType from '@rushstack/set-webpack-public-path-plugin';
+import { NewlineKind } from '@rushstack/node-core-library';
 
 import { Constants } from './utilities/Constants';
 import { LocalizationPlugin } from './LocalizationPlugin';
@@ -15,13 +16,15 @@ export interface IWebpackConfigurationUpdaterOptions {
   configuration: Webpack.Configuration;
   filesToIgnore: Set<string>;
   localeNameOrPlaceholder: string;
+  resxNewlineNormalization: NewlineKind | undefined;
 }
 
 export class WebpackConfigurationUpdater {
   public static amendWebpackConfigurationForMultiLocale(options: IWebpackConfigurationUpdaterOptions): void {
     const loader: string = path.resolve(__dirname, 'loaders', 'LocLoader.js');
     const loaderOptions: ILocLoaderOptions = {
-      pluginInstance: options.pluginInstance
+      pluginInstance: options.pluginInstance,
+      resxNewlineNormalization: options.resxNewlineNormalization
     };
 
     WebpackConfigurationUpdater._addLoadersForLocFiles(options, loader, loaderOptions);
@@ -31,7 +34,9 @@ export class WebpackConfigurationUpdater {
 
   public static amendWebpackConfigurationForInPlaceLocFiles(options: IWebpackConfigurationUpdaterOptions): void {
     const loader: string = path.resolve(__dirname, 'loaders', 'InPlaceLocFileLoader.js');
-    const loaderOptions: IBaseLoaderOptions = {}
+    const loaderOptions: IBaseLoaderOptions = {
+      resxNewlineNormalization: options.resxNewlineNormalization
+    };
 
     WebpackConfigurationUpdater._addRulesToConfiguration(
       options.configuration,

webpack/localization-plugin/src/interfaces.ts

@@ -115,6 +115,11 @@ export interface ILocalizedData {
    * Options for pseudo-localization.
    */
   pseudolocales?: IPseudolocalesOptions;
+
+  /**
+   * Normalize newlines in RESX files to either CRLF (Windows-style) or LF ('nix style)
+   */
+  normalizeResxNewlines?: 'lf' | 'crlf';
 }
 
 /**

webpack/localization-plugin/src/loaders/InPlaceLocFileLoader.ts

@@ -6,19 +6,30 @@ import { Terminal } from '@rushstack/node-core-library';
 
 import { ILocalizationFile } from '../interfaces';
 import { LocFileParser } from '../utilities/LocFileParser';
-import { loaderFactory } from './LoaderFactory';
+import {
+  loaderFactory,
+  IBaseLoaderOptions
+} from './LoaderFactory';
 import { LoaderTerminalProvider } from '../utilities/LoaderTerminalProvider';
 
-export default loaderFactory(function (this: loader.LoaderContext, locFilePath: string, content: string) {
-  const locFileData: ILocalizationFile = LocFileParser.parseLocFile({
-    content,
-    filePath: locFilePath,
-    terminal: new Terminal(LoaderTerminalProvider.getTerminalProviderForLoader(this))
-  });
-  const resultObject: { [stringName: string]: string } = {};
-  for (const stringName in locFileData) { // eslint-disable-line guard-for-in
-    resultObject[stringName] = locFileData[stringName].value;
-  }
+export default loaderFactory(
+  function (
+    this: loader.LoaderContext,
+    locFilePath: string,
+    content: string,
+    options: IBaseLoaderOptions
+  ) {
+    const locFileData: ILocalizationFile = LocFileParser.parseLocFile({
+      content,
+      filePath: locFilePath,
+      terminal: new Terminal(LoaderTerminalProvider.getTerminalProviderForLoader(this)),
+      resxNewlineNormalization: options.resxNewlineNormalization
+    });
+    const resultObject: { [stringName: string]: string } = {};
+    for (const stringName in locFileData) { // eslint-disable-line guard-for-in
+      resultObject[stringName] = locFileData[stringName].value;
+    }
 
-  return resultObject;
-});
+    return resultObject;
+  }
+);

webpack/localization-plugin/src/loaders/LoaderFactory.ts

@@ -3,8 +3,11 @@
 
 import { loader } from 'webpack';
 import * as loaderUtils from 'loader-utils';
+import { NewlineKind } from '@rushstack/node-core-library';
 
-export interface IBaseLoaderOptions { }
+export interface IBaseLoaderOptions {
+  resxNewlineNormalization: NewlineKind | undefined;
+}
 
 export interface ILoaderResult {
   [stringName: string]: string

webpack/localization-plugin/src/loaders/LocLoader.ts

@@ -30,7 +30,8 @@ export default loaderFactory(
     const locFileData: ILocalizationFile = LocFileParser.parseLocFile({
       content,
       terminal,
-      filePath: locFilePath
+      filePath: locFilePath,
+      resxNewlineNormalization: options.resxNewlineNormalization
     });
     const { additionalLoadedFilePaths, errors } = pluginInstance.addDefaultLocFile(terminal, locFilePath, locFileData);
     for (const additionalFile of additionalLoadedFilePaths) {

webpack/localization-plugin/src/utilities/LocFileParser.ts

@@ -2,7 +2,10 @@
 // See LICENSE in the project root for license information.
 
 import * as jju from 'jju';
-import { Terminal } from '@rushstack/node-core-library';
+import {
+  Terminal,
+  NewlineKind
+} from '@rushstack/node-core-library';
 
 import { ILocalizationFile } from '../interfaces';
 import { ResxReader } from './ResxReader';
@@ -15,6 +18,7 @@ export interface IParseLocFileOptions {
   terminal: Terminal;
   filePath: string;
   content: string;
+  resxNewlineNormalization: NewlineKind | undefined;
 }
 
 interface IParseCacheEntry {
@@ -29,8 +33,9 @@ const parseCache: Map<string, IParseCacheEntry> = new Map<string, IParseCacheEnt
  */
 export class LocFileParser {
   public static parseLocFile(options: IParseLocFileOptions): ILocalizationFile {
-    if (parseCache.has(options.filePath)) {
-      const entry: IParseCacheEntry = parseCache.get(options.filePath)!;
+    const fileCacheKey: string = `${options.filePath}?${options.resxNewlineNormalization || 'none'}`;
+    if (parseCache.has(fileCacheKey)) {
+      const entry: IParseCacheEntry = parseCache.get(fileCacheKey)!;
       if (entry.content === options.content) {
         return entry.parsedFile;
       }
@@ -42,7 +47,8 @@ export class LocFileParser {
         options.content,
         {
           terminal: options.terminal,
-          resxFilePath: options.filePath
+          resxFilePath: options.filePath,
+          newlineNormalization: options.resxNewlineNormalization
         }
       );
     } else {
@@ -54,7 +60,7 @@ export class LocFileParser {
       }
     }
 
-    parseCache.set(options.filePath, { content: options.content, parsedFile });
+    parseCache.set(fileCacheKey, { content: options.content, parsedFile });
     return parsedFile;
   }
 }

webpack/localization-plugin/src/utilities/ResxReader.ts

@@ -3,7 +3,9 @@
 
 import {
   FileSystem,
-  Terminal
+  Terminal,
+  Text,
+  NewlineKind
 } from '@rushstack/node-core-library';
 import {
   XmlDocument,
@@ -20,6 +22,7 @@ const STRING_NAME_RESX: RegExp = /^[A-z_$][A-z0-9_$]*$/;
 export interface IResxReaderOptions {
   resxFilePath: string;
   terminal: Terminal;
+  newlineNormalization: NewlineKind | undefined
 }
 
 interface ILoggingFunctions {
@@ -33,9 +36,9 @@ interface IResxReaderOptionsInternal {
   resxFilePath: string;
   resxContents: string;
   loggingFunctions: ILoggingFunctions;
+  newlineNormalization: NewlineKind | undefined
 }
 
-
 export class ResxReader {
   public static readResxFileAsLocFile(options: IResxReaderOptions): ILocalizationFile {
     const resxContents: string = FileSystem.readFile(options.resxFilePath);
@@ -71,7 +74,8 @@ export class ResxReader {
     return this._readResxAsLocFileInternal({
       resxFilePath: options.resxFilePath,
       resxContents,
-      loggingFunctions
+      loggingFunctions,
+      newlineNormalization: options.newlineNormalization
     })
   }
 
@@ -173,6 +177,9 @@ export class ResxReader {
               } else {
                 foundValueElement = true;
                 value = ResxReader._readTextElement(options, childNode);
+                if (value && options.newlineNormalization) {
+                  value = Text.convertTo(value, options.newlineNormalization);
+                }
               }
 
               break;