feat: consider external end import css charset

remy · Mar 28, 2017 · ef5433c · ef5433c
1 parent b87a34d
commit ef5433c
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 1 deletion.
diff --git a/lib/css.js b/lib/css.js
@@ -8,6 +8,9 @@ module.exports = {
 var Promise = require('es6-promise').Promise; // jshint ignore:line
 var debug = require('debug')('inliner');
 var basename = require('path').basename;
+var CHARTSET_RE = /@charset\s+["']([\w\-_]+)["'];/i;
+var iconv = require('iconv-lite');
+
 
 function getImages(root, css) {
   var inliner = this;
@@ -46,6 +49,7 @@ function replace(body, source, target) {
 }
 
 function getImports(root, css) {
+  css = convertCharset(css);
   // change to a string in case the CSS is a buffer, which is the case
   // when we're reading off the local file system
   if (typeof css !== 'string') {
@@ -66,7 +70,7 @@ function getImports(root, css) {
     // if url has a length > 1, then we have media types to target
     var resolvedURL = inliner.resolve(root, url[0]);
     return inliner.get(resolvedURL).then(function then(res) {
-      var importedCSS = res.body;
+      var importedCSS = convertCharset(res.body);
       inliner.jobs.done.links();
       inliner.emit('progress', 'import ' + basename(resolvedURL));
       if (url.length > 1) {
@@ -98,3 +102,26 @@ function compress(css) {
     .replace(/; /g, ';')
     .replace(/\n+/g, '');
 }
+
+function convertCharset(cssBody) {
+  var end = 0;
+  var maxLength = 64;
+  if (cssBody) {
+    end = cssBody.length > maxLength ? maxLength : cssBody.length;
+    var charsetData = cssBody.slice(0, end).toString();
+    var matchs = CHARTSET_RE.exec(charsetData);
+    if (matchs) {
+      var charset = matchs[1].toLowerCase();
+      if (charset && charset !== 'utf-8' && charset !== 'utf8') {
+        debug('decoding from: %s', charset);
+        if (iconv.encodingExists(charset)) {
+          return iconv.encode(iconv.decode(cssBody, charset), 'utf-8')
+            .toString()
+            .replace(CHARTSET_RE, '');
+        }
+        console.warn('no such charset: ' + charset);
+      }
+    }
+  }
+  return cssBody;
+}
diff --git a/test/fixtures/css-ext-charset-import.css b/test/fixtures/css-ext-charset-import.css
@@ -0,0 +1 @@
+@charset "shift_jis";.body2 {width:100%;font-family: "ƒƒCƒŠƒI", "ƒqƒ‰ƒMƒmŠpƒS Pro W3", "MS PGothic", "MS UI Gothic", Helvetica, Arial, sans-serif;}
diff --git a/test/fixtures/css-ext-charset.css b/test/fixtures/css-ext-charset.css
@@ -0,0 +1,6 @@
+@charset "shift_jis";
+@import url('css-ext-charset-import.css');
+
+body {
+    font-family: "ƒƒCƒŠƒI", "ƒqƒ‰ƒMƒmŠpƒS Pro W3", "MS PGothic", "MS UI Gothic", Helvetica, Arial, sans-serif;
+}
diff --git a/test/fixtures/css-ext-charset.result.html b/test/fixtures/css-ext-charset.result.html
@@ -0,0 +1 @@
+<!DOCTYPE html> <html> <head> <title>External css and multibyte characters - 日本語</title> <style> .body2{width:100%;font-family:"メイリオ", "ヒラギノ角ゴ Pro W3", "MS PGothic", "MS UI Gothic", Helvetica, Arial, sans-serif;};body{ font-family:"メイリオ", "ヒラギノ角ゴ Pro W3", "MS PGothic", "MS UI Gothic", Helvetica, Arial, sans-serif;}</style> </head> <body> </body> </html>
diff --git a/test/fixtures/css-ext-charset.src.html b/test/fixtures/css-ext-charset.src.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="shift_jis">
+  <title>External css and multibyte characters - “ú–{Œê</title>
+  <link href="http://localhost:54321/css-ext-charset.css" rel="stylesheet" type="text/css" />
+</head>
+<body>
+
+</body>
+</html>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		@charset "shift_jis";.body2 {width:100%;font-family: "ƒƒCƒŠƒI", "ƒqƒ‰ƒMƒmŠpƒS Pro W3", "MS PGothic", "MS UI Gothic", Helvetica, Arial, sans-serif;}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		<!DOCTYPE html> <html> <head> <title>External css and multibyte characters - 日本語</title> <style> .body2{width:100%;font-family:"メイリオ", "ヒラギノ角ゴ Pro W3", "MS PGothic", "MS UI Gothic", Helvetica, Arial, sans-serif;};body{ font-family:"メイリオ", "ヒラギノ角ゴ Pro W3", "MS PGothic", "MS UI Gothic", Helvetica, Arial, sans-serif;}</style> </head> <body> </body> </html>