Merge pull request #47 from embulk/enhance-base64encode-method

enhance base64 encode method to support Japanese character

Author: hieudion

.travis.yml

@@ -1,5 +1,7 @@
 language: java
 
+dist: trusty
+
 jdk:
   - oraclejdk8
 

src/main/java/org/embulk/input/gcs/GcsFileInput.java

@@ -80,17 +80,25 @@ static FileList listFiles(PluginTask task)
         return builder.build();
     }
 
-    // String nextToken = base64Encode(0x0a + 0x01~0x27 + filePath);
+    // String nextToken = base64Encode(0x0a + ASCII character according to utf8EncodeLength position+ filePath);
     @VisibleForTesting
     static String base64Encode(String path)
     {
         byte[] encoding;
         byte[] utf8 = path.getBytes(Charsets.UTF_8);
         LOG.debug("path string: {} ,path length:{} \" + ", path, utf8.length);
 
+        int utf8EncodeLength = utf8.length;
+        if (utf8EncodeLength >= 128) {
+            throw new ConfigException(String.format("last_path '%s' is too long to encode. Please try to reduce its length", path));
+        }
+
         encoding = new byte[utf8.length + 2];
         encoding[0] = 0x0a;
-        encoding[1] = Byte.valueOf(String.valueOf(path.length()));
+
+        // for example: 60 -> '<'
+        char temp = (char) utf8EncodeLength;
+        encoding[1] = (byte) temp;
         System.arraycopy(utf8, 0, encoding, 2, utf8.length);
 
         String s = BaseEncoding.base64().encode(encoding);

src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -346,6 +346,23 @@ public void testNonExistingFilesWithPaths() throws Exception
         runner.transaction(config, new Control());
     }
 
+    @Test(expected = ConfigException.class)
+    public void testLastPathTooLong() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("paths", Arrays.asList())
+                .set("auth_method", "private_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("p12_keyfile", GCP_P12_KEYFILE)
+                .set("json_keyfile", GCP_JSON_KEYFILE)
+                .set("application_name", GCP_APPLICATION_NAME)
+                .set("last_path", "テストダミー/テストダミーテストダミーテストダミーテストダミーテストダミーテストダミーテストダミー.csv")
+                .set("parser", parserConfig(schemaConfig()));
+
+        runner.transaction(config, new Control());
+    }
+
     @Test
     public void testGcsFileInputByOpen()
     {
@@ -375,6 +392,10 @@ public void testBase64()
         String params = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc127";
         String expected = "Cn9jY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjMTI3";
         assertEquals(expected, GcsFileInput.base64Encode(params));
+
+        params = "テストダミー/テス123/テストダミー/テストダミ.csv";
+        expected = "CkPjg4bjgrnjg4jjg4Djg5/jg7wv44OG44K5MTIzL+ODhuOCueODiOODgOODn+ODvC/jg4bjgrnjg4jjg4Djg58uY3N2";
+        assertEquals(expected, GcsFileInput.base64Encode(params));
     }
 
     private ConfigSource config()