Support nesting non-capturing regexp groups, and expose utility for 'NormalizePathname' behavior

Author: guoyao

README.md

@@ -21,7 +21,10 @@ import pathToRegexp "github.com/soongo/path-to-regexp"
 // pathToRegexp.MustCompile(path, options) // like Compile but panics if the error is non-nil
 // pathToRegexp.Match(path, options) // options can be nil
 // pathToRegexp.MustMatch(path, options) // like Match but panics if the error is non-nil
-// pathToRegexp.Must(regexp, err) // wraps a call to a function returning (*regexp2.Regexp, error) and panics if the error is non-nil.
+// pathToRegexp.Must(regexp, err) // wraps a call to a function returning (*regexp2.Regexp, error) and panics if the error is non-nil
+// pathToRegexp.EncodeURI(str) // encodes characters in URI, like javascript's encodeURI
+// pathToRegexp.EncodeURIComponent(str) // encodes characters in URI except `;/?:@&=+$,#`, like javascript's encodeURIComponent
+// pathToRegexp.NormalizePathname(str) // return a normalized string
 ```
 
 - **path** A string, array or slice of strings, or a regular expression with type *github.com/dlclark/regexp2.Regexp.
@@ -217,6 +220,16 @@ fmt.Printf("%#v\n", match("/user/123"))
 match("/invalid") //=> nil
 ```
 
+### Normalize Pathname
+
+The `NormalizePathname` function will return a normalized string for matching with `PathToRegexp`.
+
+```js
+re := pathToRegexp.Must(pathToRegexp.PathToRegexp("/caf\u00E9", nil, nil))
+input := pathToRegexp.EncodeURI("/cafe\u0301");
+re.MatchString(pathToRegexp.NormalizePathname(input)); //=> true, nil
+```
+
 ### Parse
 
 The `Parse` function will return a list of strings and tokens from a path string:

go.mod

@@ -2,4 +2,7 @@ module github.com/soongo/path-to-regexp
 
 go 1.13
 
-require github.com/dlclark/regexp2 v1.2.0
+require (
+	github.com/dlclark/regexp2 v1.2.0
+	golang.org/x/text v0.3.2
+)

go.sum

@@ -1,2 +1,5 @@
 github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk=
 github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

path_to_regexp.go

@@ -11,9 +11,14 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
+	"unicode"
 	"unsafe"
 
+	"golang.org/x/text/runes"
+
 	"github.com/dlclark/regexp2"
+	"golang.org/x/text/transform"
+	"golang.org/x/text/unicode/norm"
 )
 
 // Token is parsed from path. For example, using `/user/:id`, `tokens` will
@@ -88,6 +93,53 @@ const defaultDelimiter = "/"
 var escapeRegexp = regexp2.MustCompile("([.+*?=^!:${}()[\\]|/\\\\])", regexp2.None)
 var tokenRegexp = regexp2.MustCompile("\\((?!\\?)", regexp2.None)
 
+func normalize(str string) string {
+	t := transform.Chain(norm.NFC, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
+	normStr, _, _ := transform.String(t, str)
+	return normStr
+}
+
+// NormalizePathname normalizes a pathname for matching, replaces multiple slashes
+// with a single slash and normalizes unicode characters to "NFC". When using this method,
+// `decode` should be an identity function so you don't decode strings twice.
+func NormalizePathname(pathname string) string {
+	r := regexp2.MustCompile("\\/+", regexp2.None)
+	str, err := r.Replace(DecodeURIComponent(pathname, nil),
+		"/", -1, -1)
+	if err != nil {
+		panic(err)
+	}
+	return normalize(str)
+}
+
+// Balanced bracket helper function.
+func balanced(open string, close string, str string, index int) int {
+	count, i, arr := 0, index, strings.Split(str, "")
+
+	for i < len(arr) {
+		if arr[i] == "\\" {
+			i += 2
+			continue
+		}
+
+		if arr[i] == close {
+			count--
+
+			if count == 0 {
+				return i + 1
+			}
+		}
+
+		if arr[i] == open {
+			count++
+		}
+
+		i++
+	}
+
+	return -1
+}
+
 // Parse a string for the raw tokens.
 func Parse(str string, o *Options) []interface{} {
 	tokens, tokenIndex, index, path, isEscaped := make([]interface{}, 0), 0, 0, "", false
@@ -136,49 +188,27 @@ func Parse(str string, o *Options) []interface{} {
 		}
 
 		if index < length && arr[index] == "(" {
-			prev, balanced, invalidGroup := index, 1, false
-			if index+1 < length && arr[index+1] == "?" {
-				panic("Path pattern must be a capturing group")
-			}
-
-			for index++; index < length; index++ {
-				if arr[index] == "\\" {
-					pattern += strings.Join(arr[index:min(index+2, length)], "")
-					index++
-					continue
+			end := balanced("(", ")", str, index)
+
+			// False positive on matching brackets.
+			if end > -1 {
+				pattern = strings.Join(arr[index+1:end-1], "")
+				index = end
+				if pattern[0] == '?' {
+					panic("Path pattern must be a capturing group")
 				}
 
-				if arr[index] == ")" {
-					balanced--
-					if balanced == 0 {
-						index++
-						break
+				r := regexp2.MustCompile("\\((?=[^?])", regexp2.None)
+				if ok, _ := r.MatchString(pattern); ok {
+					validPattern, err := r.Replace(pattern, "(?:", -1, -1)
+					if err != nil {
+						panic(err)
 					}
-				}
 
-				pattern += string(arr[index])
-
-				if arr[index] == "(" {
-					balanced++
-
-					// Better errors on nested capturing groups.
-					if index+1 >= length || arr[index+1] != "?" {
-						pattern += "?:"
-						invalidGroup = true
-					}
+					panic(fmt.Sprintf("Capturing groups are not allowed in pattern, "+
+						"use a non-capturing group: (%s)", validPattern))
 				}
 			}
-
-			if invalidGroup {
-				panic(fmt.Sprintf("Capturing groups are not allowed in pattern, "+
-					"use a non-capturing group: (%s)", pattern))
-			}
-
-			// False positive.
-			if balanced > 0 {
-				index = prev
-				pattern = ""
-			}
 		}
 
 		// Add regular characters to the path string.
@@ -271,7 +301,7 @@ func Match(path interface{}, o *Options) (func(string) *MatchResult, error) {
 		return nil, err
 	}
 
-	return regexpToFunction(re, tokens), nil
+	return regexpToFunction(re, tokens, o), nil
 }
 
 // MustMatch is like Match but panics if err occur in match function.
@@ -284,8 +314,13 @@ func MustMatch(path interface{}, o *Options) func(string) *MatchResult {
 }
 
 // Create a path match function from `path-to-regexp` output.
-func regexpToFunction(re *regexp2.Regexp, tokens []Token) func(string) *MatchResult {
-	decode := DecodeURIComponent
+func regexpToFunction(re *regexp2.Regexp, tokens []Token, o *Options) func(string) *MatchResult {
+	decode := func(str string, token interface{}) string {
+		return str
+	}
+	if o != nil && o.Decode != nil {
+		decode = o.Decode
+	}
 
 	return func(pathname string) *MatchResult {
 		m, err := re.FindStringMatch(pathname)
@@ -513,6 +548,20 @@ func toMap(data interface{}) map[interface{}]interface{} {
 	return m
 }
 
+func EncodeURI(str string) string {
+	excludes := ";/?:@&=+$,#"
+	arr := strings.Split(str, "")
+	result := ""
+	for _, v := range arr {
+		if strings.Contains(excludes, v) {
+			result += v
+		} else {
+			result += EncodeURIComponent(v, nil)
+		}
+	}
+	return result
+}
+
 func EncodeURIComponent(str string, token interface{}) string {
 	r := url.QueryEscape(str)
 	r = strings.Replace(r, "+", "%20", -1)

path_to_regexp_test.go

@@ -202,6 +202,17 @@ var tests = []a{
 				a{"/route", "route"},
 				&MatchResult{Path: "/route", Index: 0, Params: m{"test": "route"}},
 			},
+			a{
+				"/caf%C3%A9",
+				a{"/caf%C3%A9", "caf%C3%A9"},
+				&MatchResult{Path: "/caf%C3%A9", Index: 0, Params: m{"test": "caf%C3%A9"}},
+			},
+			a{
+				"/caf%C3%A9",
+				a{"/caf%C3%A9", "caf%C3%A9"},
+				&MatchResult{Path: "/caf%C3%A9", Index: 0, Params: m{"test": "café"}},
+				&Options{Decode: DecodeURIComponent},
+			},
 		},
 		a{
 			a{m{}, nil},
@@ -2774,14 +2785,14 @@ var tests = []a{
 	},
 
 	/**
-	 * Nested parenthesis.
+	 * Nested parentheses.
 	 */
 	{
-		"/:foo(\\d+(?:\\.\\d+)?)",
-		&Options{},
+		"/:test(\\d+(?:\\.\\d+)?)",
+		nil,
 		a{
 			Token{
-				Name:      "foo",
+				Name:      "test",
 				Prefix:    "/",
 				Delimiter: "/",
 				Optional:  false,
@@ -2791,12 +2802,40 @@ var tests = []a{
 		},
 		a{
 			a{"/123", a{"/123", "123"}},
+			a{"/abc", nil},
+			a{"/123/abc", nil},
 			a{"/123.123", a{"/123.123", "123.123"}},
+			a{"/123.abc", nil},
+		},
+		a{
+			a{m{"test": 123}, "/123"},
+			a{m{"test": 123.123}, "/123.123"},
+			a{m{"test": "abc"}, nil},
+			a{m{"test": "123"}, "/123"},
+			a{m{"test": "123.123"}, "/123.123"},
+			a{m{"test": "123.abc"}, nil},
+		},
+	},
+	{
+		"/:test((?!login)[^/]+)",
+		nil,
+		a{
+			Token{
+				Name:      "test",
+				Prefix:    "/",
+				Delimiter: "/",
+				Optional:  false,
+				Repeat:    false,
+				Pattern:   "(?!login)[^/]+",
+			},
 		},
 		a{
-			a{m{"foo": 123}, "/123"},
-			a{m{"foo": 123.123}, "/123.123"},
-			a{m{"foo": "123"}, "/123"},
+			a{"/route", a{"/route", "route"}},
+			a{"/login", nil},
+		},
+		a{
+			a{m{"test": "route"}, "/route"},
+			a{m{"test": "login"}, nil},
 		},
 	},
 }
@@ -3027,8 +3066,13 @@ func TestPathToRegexp(t *testing.T) {
 							params = io[2].(*MatchResult)
 						}
 
+						var options *Options
+						if len(io) >= 4 && io[3] != nil {
+							options = io[3].(*Options)
+						}
+
 						if path, ok := path.(string); ok && params != nil {
-							match := MustMatch(path, nil)
+							match := MustMatch(path, options)
 							t.Run(message+" params", func(t *testing.T) {
 								m := match(pathname.(string))
 								if !params.equals(m) {
@@ -3119,6 +3163,24 @@ func TestPathToRegexp(t *testing.T) {
 		})
 	})
 
+	t.Run("normalize pathname", func(t *testing.T) {
+		t.Run("should match normalized pathnames", func(t *testing.T) {
+			re := Must(PathToRegexp("/caf\u00E9", nil, nil))
+			input := EncodeURI("/cafe\u0301")
+
+			result := exec(re, input)
+			if result != nil {
+				t.Errorf("got %v want %v", result, nil)
+			}
+
+			want := []string{"/caf\u00E9"}
+			result = exec(re, NormalizePathname(input))
+			if !reflect.DeepEqual(result, want) {
+				t.Errorf("got %v want %v", result, want)
+			}
+		})
+	})
+
 	t.Run("path should be string, or strings, or a regular expression", func(t *testing.T) {
 		_, err := PathToRegexp(123, nil, nil)
 		if err == nil {