Make allow_malformed option apply to url decoding

Author: jlinn

src/main/java/org/elasticsearch/index/analysis/url/URLTokenizer.java

@@ -196,7 +196,13 @@ private List<Token> tokenize(URL url, URLPart part) throws IOException {
         int start = 0;
         int end = 0;
         if (urlDecode) {
-            partString = URLDecoder.decode(partString, "UTF-8");
+            try {
+                partString = URLDecoder.decode(partString, "UTF-8");
+            } catch (IllegalArgumentException e) {
+                if (!allowMalformed) {
+                    throw new IOException("Error performing URL decoding on string: " + partString, e);
+                }
+            }
         }
         switch (part) {
             case HOST:

src/test/java/org/elasticsearch/index/analysis/url/URLTokenizerTest.java

@@ -147,6 +147,34 @@ public void testAllowMalformed() throws IOException {
     }
 
 
+    @Test
+    public void testUrlDecode() throws Exception {
+        String url = "http://foo.com?baz=foo%20bat";
+        URLTokenizer tokenizer = createTokenizer(url, URLPart.QUERY);
+        tokenizer.setUrlDecode(true);
+        assertTokenStreamContents(tokenizer, stringArray("baz=foo bat"));
+    }
+
+
+    @Test(expected = IOException.class)
+    public void testUrlDecodeIllegalCharacters() throws Exception {
+        String url = "http://foo.com?baz=foo%2vbat";
+        URLTokenizer tokenizer = createTokenizer(url, URLPart.QUERY);
+        tokenizer.setUrlDecode(true);
+        assertTokenStreamContents(tokenizer, "");
+    }
+
+
+    @Test
+    public void testUrlDecodeAllowMalformed() throws Exception {
+        String url = "http://foo.com?baz=foo%2vbat";
+        URLTokenizer tokenizer = createTokenizer(url, URLPart.QUERY);
+        tokenizer.setUrlDecode(true);
+        tokenizer.setAllowMalformed(true);
+        assertTokenStreamContents(tokenizer, "baz=foo%2vbat");
+    }
+
+
     private URLTokenizer createTokenizer(String input, URLPart part) throws IOException {
         URLTokenizer tokenizer = new URLTokenizer(part);
         tokenizer.setReader(new StringReader(input));