From e69798d909678ac6fb938baf10245c117d248e64 Mon Sep 17 00:00:00 2001
From: Ehryk <rhaistlin@gmail.com>
Date: Sat, 1 Aug 2015 00:58:12 -0500
Subject: [PATCH] Added multiple tokenizers implementing new ITokenizer
 interface (GetOpt = previous, Windows, Hybrid)

---
 .../Unit/Core/InstanceBuilderTests.cs         |   5 +-
 .../Unit/Core/TokenizerTests.cs               |   6 +-
 src/CommandLine/CommandLine.csproj            |   6 +-
 src/CommandLine/Core/ITokenizer.cs            |  26 ++
 src/CommandLine/Core/InstanceBuilder.cs       |   6 +-
 src/CommandLine/Core/InstanceChooser.cs       |   6 +-
 .../Core/{Tokenizer.cs => TokenizerGetOpt.cs} | 272 +++++++++---------
 src/CommandLine/Core/TokenizerHybrid.cs       | 143 +++++++++
 src/CommandLine/Core/TokenizerWindows.cs      | 137 +++++++++
 src/CommandLine/Parser.cs                     |  10 +-
 10 files changed, 468 insertions(+), 149 deletions(-)
 create mode 100644 src/CommandLine/Core/ITokenizer.cs
 rename src/CommandLine/Core/{Tokenizer.cs => TokenizerGetOpt.cs} (92%)
 create mode 100644 src/CommandLine/Core/TokenizerHybrid.cs
 create mode 100644 src/CommandLine/Core/TokenizerWindows.cs

diff --git a/src/CommandLine.Tests/Unit/Core/InstanceBuilderTests.cs b/src/CommandLine.Tests/Unit/Core/InstanceBuilderTests.cs
index 5a65c90a..c1bbcd7c 100644
--- a/src/CommandLine.Tests/Unit/Core/InstanceBuilderTests.cs
+++ b/src/CommandLine.Tests/Unit/Core/InstanceBuilderTests.cs
@@ -435,13 +435,14 @@ public void Double_dash_force_subsequent_arguments_as_values()
                     IntValue = 20
                 };
             var arguments = new[] { "--stringvalue", "str1", "--", "10", "-a", "--bee", "-c", "20" };
+            var tokenizer = new TokenizerGetOpt();
 
             // Exercize system 
             var result = InstanceBuilder.Build(
                 Maybe.Just<Func<FakeOptionsWithValues>>(() => new FakeOptionsWithValues()),
                 (a, optionSpecs) =>
-                    Tokenizer.PreprocessDashDash(a,
-                        args => Tokenizer.Tokenize(args, name => NameLookup.Contains(name, optionSpecs, StringComparer.Ordinal))),
+                    tokenizer.Preprocess(a,
+                        args => tokenizer.Tokenize(args, name => NameLookup.Contains(name, optionSpecs, StringComparer.Ordinal))),
                 arguments,
                 StringComparer.Ordinal,
                 CultureInfo.InvariantCulture);
diff --git a/src/CommandLine.Tests/Unit/Core/TokenizerTests.cs b/src/CommandLine.Tests/Unit/Core/TokenizerTests.cs
index 655fdcf8..d3844928 100644
--- a/src/CommandLine.Tests/Unit/Core/TokenizerTests.cs
+++ b/src/CommandLine.Tests/Unit/Core/TokenizerTests.cs
@@ -23,10 +23,11 @@ public void Explode_scalar_with_separator_in_odd_args_input_returns_sequence()
                 Token.Value("aaa"), Token.Value("bb"),  Token.Value("cccc"), Token.Name("switch") };
             var specs = new[] { new OptionSpecification(string.Empty, "string-seq",
                 false, string.Empty, Maybe.Nothing<int>(), Maybe.Nothing<int>(), ',', null, string.Empty, string.Empty, new List<string>(), typeof(IEnumerable<string>), TargetType.Sequence)};
+            var tokenizer = new TokenizerGetOpt();
 
             // Exercize system
             var result =
-                Tokenizer.ExplodeOptionList(
+                tokenizer.ExplodeOptionList(
                     Result.Succeed(
                         Enumerable.Empty<Token>().Concat(new[] { Token.Name("i"), Token.Value("10"),
                             Token.Name("string-seq"), Token.Value("aaa,bb,cccc"), Token.Name("switch") }),
@@ -47,10 +48,11 @@ public void Explode_scalar_with_separator_in_even_args_input_returns_sequence()
                 Token.Value("aaa"), Token.Value("bb"),  Token.Value("cccc"), Token.Name("switch") };
             var specs = new[] { new OptionSpecification(string.Empty, "string-seq",
                 false, string.Empty, Maybe.Nothing<int>(), Maybe.Nothing<int>(), ',', null, string.Empty, string.Empty, new List<string>(), typeof(IEnumerable<string>), TargetType.Sequence)};
+            var tokenizer = new TokenizerGetOpt();
 
             // Exercize system
             var result =
-                Tokenizer.ExplodeOptionList(
+                tokenizer.ExplodeOptionList(
                     Result.Succeed(
                         Enumerable.Empty<Token>().Concat(new[] { Token.Name("x"),
                             Token.Name("string-seq"), Token.Value("aaa,bb,cccc"), Token.Name("switch") }),
diff --git a/src/CommandLine/CommandLine.csproj b/src/CommandLine/CommandLine.csproj
index 9dea8f8f..ebf27423 100644
--- a/src/CommandLine/CommandLine.csproj
+++ b/src/CommandLine/CommandLine.csproj
@@ -74,6 +74,9 @@
     <Compile Include="Core\ArgumentsExtensions.cs" />
     <Compile Include="Core\KeyValuePairHelper.cs" />
     <Compile Include="Core\PreprocessorGuards.cs" />
+    <Compile Include="Core\ITokenizer.cs" />
+    <Compile Include="Core\TokenizerHybrid.cs" />
+    <Compile Include="Core\TokenizerWindows.cs" />
     <Compile Include="Core\TypeDescriptor.cs" />
     <Compile Include="Core\Scalar.cs" />
     <Compile Include="Core\Sequence.cs" />
@@ -82,7 +85,7 @@
     <Compile Include="ErrorExtensions.cs" />
     <Compile Include="Infrastructure\EnumerableExtensions`1.cs" />
     <Compile Include="Core\InstanceChooser.cs" />
-    <Compile Include="Core\Tokenizer.cs">
+    <Compile Include="Core\TokenizerGetOpt.cs">
       <SubType>Code</SubType>
     </Compile>
     <Compile Include="Core\Verb.cs" />
@@ -148,6 +151,7 @@
     <Compile Include="ValueAttribute.cs" />
     <Compile Include="VerbAttribute.cs" />
   </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.
diff --git a/src/CommandLine/Core/ITokenizer.cs b/src/CommandLine/Core/ITokenizer.cs
new file mode 100644
index 00000000..b84d8b66
--- /dev/null
+++ b/src/CommandLine/Core/ITokenizer.cs
@@ -0,0 +1,26 @@
+// Copyright 2005-2015 Giacomo Stelluti Scala & Contributors. All rights reserved. See doc/License.md in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using CommandLine.Infrastructure;
+using CSharpx;
+using RailwaySharp.ErrorHandling;
+
+namespace CommandLine.Core
+{
+    internal interface ITokenizer
+    {
+        Result<IEnumerable<Token>, Error> Tokenize(
+            IEnumerable<string> arguments,
+            Func<string, bool> nameLookup);
+
+        Result<IEnumerable<Token>, Error> Preprocess(
+            IEnumerable<string> arguments,
+            Func<IEnumerable<string>, Result<IEnumerable<Token>, Error>> tokenizer);
+
+        Result<IEnumerable<Token>, Error> ExplodeOptionList(
+            Result<IEnumerable<Token>, Error> tokenizerResult,
+            Func<string, Maybe<char>> optionSequenceWithSeparatorLookup);
+    }
+}
\ No newline at end of file
diff --git a/src/CommandLine/Core/InstanceBuilder.cs b/src/CommandLine/Core/InstanceBuilder.cs
index 99dbdf81..23a4f9e0 100644
--- a/src/CommandLine/Core/InstanceBuilder.cs
+++ b/src/CommandLine/Core/InstanceBuilder.cs
@@ -18,12 +18,14 @@ public static ParserResult<T> Build<T>(
             StringComparer nameComparer,
             CultureInfo parsingCulture)
         {
+            var tokenizer = new TokenizerGetOpt();
+
             return Build(
                 factory,
                 (args, optionSpecs) =>
                     {
-                        var tokens = Tokenizer.Tokenize(args, name => NameLookup.Contains(name, optionSpecs, nameComparer));
-                        var explodedTokens = Tokenizer.ExplodeOptionList(
+                        var tokens = tokenizer.Tokenize(args, name => NameLookup.Contains(name, optionSpecs, nameComparer));
+                        var explodedTokens = tokenizer.ExplodeOptionList(
                             tokens,
                             name => NameLookup.HavingSeparator(name, optionSpecs, nameComparer));
                         return explodedTokens;
diff --git a/src/CommandLine/Core/InstanceChooser.cs b/src/CommandLine/Core/InstanceChooser.cs
index 1b07d226..fdc3db83 100644
--- a/src/CommandLine/Core/InstanceChooser.cs
+++ b/src/CommandLine/Core/InstanceChooser.cs
@@ -18,11 +18,13 @@ public static ParserResult<object> Choose(
             StringComparer nameComparer,
             CultureInfo parsingCulture)
         {
+            var tokenizer = new TokenizerGetOpt();
+
             return Choose(
                 (args, optionSpecs) =>
                     {
-                        var tokens = Tokenizer.Tokenize(args, name => NameLookup.Contains(name, optionSpecs, nameComparer));
-                        var explodedTokens = Tokenizer.ExplodeOptionList(tokens, name => NameLookup.HavingSeparator(name, optionSpecs, nameComparer));
+                        var tokens = tokenizer.Tokenize(args, name => NameLookup.Contains(name, optionSpecs, nameComparer));
+                        var explodedTokens = tokenizer.ExplodeOptionList(tokens, name => NameLookup.HavingSeparator(name, optionSpecs, nameComparer));
                         return explodedTokens;
                     },
                 types,
diff --git a/src/CommandLine/Core/Tokenizer.cs b/src/CommandLine/Core/TokenizerGetOpt.cs
similarity index 92%
rename from src/CommandLine/Core/Tokenizer.cs
rename to src/CommandLine/Core/TokenizerGetOpt.cs
index df400892..e4703260 100644
--- a/src/CommandLine/Core/Tokenizer.cs
+++ b/src/CommandLine/Core/TokenizerGetOpt.cs
@@ -1,137 +1,137 @@
-// Copyright 2005-2015 Giacomo Stelluti Scala & Contributors. All rights reserved. See doc/License.md in the project root for license information.
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using CommandLine.Infrastructure;
-using CSharpx;
-using RailwaySharp.ErrorHandling;
-
-namespace CommandLine.Core
-{
-    internal static class Tokenizer
-    {
-        public static Result<IEnumerable<Token>, Error> Tokenize(
-            IEnumerable<string> arguments,
-            Func<string, bool> nameLookup)
-        {
-            var errors = new List<Error>();
-            Action<Error> onError = errors.Add;
-
-            var tokens = (from arg in arguments
-                          from token in !arg.StartsWith("-", StringComparison.Ordinal)
-                               ? new[] { Token.Value(arg) }
-                               : arg.StartsWith("--", StringComparison.Ordinal)
-                                     ? TokenizeLongName(arg, onError)
-                                     : TokenizeShortName(arg, nameLookup)
-                          select token).Memorize();
-
-            var unkTokens = (from t in tokens where t.IsName() && !nameLookup(t.Text) select t).Memorize();
-
-            return Result.Succeed(tokens.Where(x => !unkTokens.Contains(x)), errors.Concat(from t in unkTokens select new UnknownOptionError(t.Text)));
-        }
-
-        public static Result<IEnumerable<Token>, Error> PreprocessDashDash(
-            IEnumerable<string> arguments,
-            Func<IEnumerable<string>, Result<IEnumerable<Token>, Error>> tokenizer)
-        {
-            if (arguments.Any(arg => arg.EqualsOrdinal("--")))
-            {
-                var tokenizerResult = tokenizer(arguments.TakeWhile(arg => !arg.EqualsOrdinal("--")));
-                var values = arguments.SkipWhile(arg => !arg.EqualsOrdinal("--")).Skip(1).Select(Token.Value);
-                return tokenizerResult.Map(tokens => tokens.Concat(values));
-            }
-            return tokenizer(arguments);
-        }
-
-        public static Result<IEnumerable<Token>, Error> ExplodeOptionList(
-            Result<IEnumerable<Token>, Error> tokenizerResult,
-            Func<string, Maybe<char>> optionSequenceWithSeparatorLookup)
-        {
-            var tokens = tokenizerResult.SucceededWith();
-
-            var replaces = tokens.Select((t,i) =>
-                optionSequenceWithSeparatorLookup(t.Text)
-                    .Return(sep => Tuple.Create(i + 1, sep),
-                        Tuple.Create(-1, '\0'))).SkipWhile(x => x.Item1 < 0);
-
-            var exploded = tokens.Select((t, i) =>
-                        replaces.FirstOrDefault(x => x.Item1 == i).ToMaybe()
-                            .Return(r => t.Text.Split(r.Item2).Select(Token.Value),
-                                Enumerable.Empty<Token>().Concat(new[]{ t })));
-
-            var flattened = exploded.SelectMany(x => x);
-
-            return Result.Succeed(flattened, tokenizerResult.SuccessfulMessages());
-        }
-
-        private static IEnumerable<Token> TokenizeShortName(
-            string value,
-            Func<string, bool> nameLookup)
-        {
-            if (value.Length > 1 || value[0] == '-' || value[1] != '-')
-            {
-                var text = value.Substring(1);
-
-                if (char.IsDigit(text[0]))
-                {
-                    yield return Token.Value(value);
-                    yield break;
-                }
-
-                if (value.Length == 2)
-                {
-                    yield return Token.Name(text);
-                    yield break;
-                }
-
-                var first = text.Substring(0, 1);
-                yield return Token.Name(first);
-
-                var seen = new List<char> { first[0] };
-
-                foreach (var c in text.Substring(1))
-                {
-                    var n = new string(c, 1);
-                    if (!seen.Contains(c) && nameLookup(n))
-                    {
-                        seen.Add(c);
-                        yield return Token.Name(n);
-                    }
-                    else
-                    {
-                        break;
-                    }
-                }
-                if (seen.Count() < text.Length)
-                {
-                    yield return Token.Value(text.Substring(seen.Count()));
-                }
-            }
-        }
-
-        private static IEnumerable<Token> TokenizeLongName(
-            string value,
-            Action<Error> onError)
-        {
-            if (value.Length > 2 && value.StartsWith("--", StringComparison.Ordinal))
-            {
-                var text = value.Substring(2);
-                var equalIndex = text.IndexOf('=');
-                if (equalIndex <= 0)
-                {
-                    yield return Token.Name(text);
-                    yield break;
-                }
-                if (equalIndex == 1) // "--="
-                {
-                    onError(new BadFormatTokenError(value));
-                    yield break;
-                }
-                var parts = text.Split('=');
-                yield return Token.Name(parts[0]);
-                yield return Token.Value(parts[1]);
-            }
-        }
-    }
+// Copyright 2005-2015 Giacomo Stelluti Scala & Contributors. All rights reserved. See doc/License.md in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using CommandLine.Infrastructure;
+using CSharpx;
+using RailwaySharp.ErrorHandling;
+
+namespace CommandLine.Core
+{
+    internal class TokenizerGetOpt : ITokenizer
+    {
+        public Result<IEnumerable<Token>, Error> Tokenize(
+            IEnumerable<string> arguments,
+            Func<string, bool> nameLookup)
+        {
+            var errors = new List<Error>();
+            Action<Error> onError = errors.Add;
+
+            var tokens = (from arg in arguments
+                          from token in !arg.StartsWith("-", StringComparison.Ordinal)
+                               ? new[] { Token.Value(arg) }
+                               : arg.StartsWith("--", StringComparison.Ordinal)
+                                     ? TokenizeLongName(arg, onError)
+                                     : TokenizeShortName(arg, nameLookup)
+                          select token).Memorize();
+
+            var unkTokens = (from t in tokens where t.IsName() && !nameLookup(t.Text) select t).Memorize();
+
+            return Result.Succeed(tokens.Where(x => !unkTokens.Contains(x)), errors.Concat(from t in unkTokens select new UnknownOptionError(t.Text)));
+        }
+
+        public Result<IEnumerable<Token>, Error> Preprocess(
+            IEnumerable<string> arguments,
+            Func<IEnumerable<string>, Result<IEnumerable<Token>, Error>> tokenizer)
+        {
+            if (arguments.Any(arg => arg.EqualsOrdinal("--")))
+            {
+                var tokenizerResult = tokenizer(arguments.TakeWhile(arg => !arg.EqualsOrdinal("--")));
+                var values = arguments.SkipWhile(arg => !arg.EqualsOrdinal("--")).Skip(1).Select(Token.Value);
+                return tokenizerResult.Map(tokens => tokens.Concat(values));
+            }
+            return tokenizer(arguments);
+        }
+
+        public Result<IEnumerable<Token>, Error> ExplodeOptionList(
+            Result<IEnumerable<Token>, Error> tokenizerResult,
+            Func<string, Maybe<char>> optionSequenceWithSeparatorLookup)
+        {
+            var tokens = tokenizerResult.SucceededWith();
+
+            var replaces = tokens.Select((t,i) =>
+                optionSequenceWithSeparatorLookup(t.Text)
+                    .Return(sep => Tuple.Create(i + 1, sep),
+                        Tuple.Create(-1, '\0'))).SkipWhile(x => x.Item1 < 0);
+
+            var exploded = tokens.Select((t, i) =>
+                        replaces.FirstOrDefault(x => x.Item1 == i).ToMaybe()
+                            .Return(r => t.Text.Split(r.Item2).Select(Token.Value),
+                                Enumerable.Empty<Token>().Concat(new[]{ t })));
+
+            var flattened = exploded.SelectMany(x => x);
+
+            return Result.Succeed(flattened, tokenizerResult.SuccessfulMessages());
+        }
+
+        private static IEnumerable<Token> TokenizeShortName(
+            string value,
+            Func<string, bool> nameLookup)
+        {
+            if (value.Length > 1 || value[0] == '-' || value[1] != '-')
+            {
+                var text = value.Substring(1);
+
+                if (char.IsDigit(text[0]))
+                {
+                    yield return Token.Value(value);
+                    yield break;
+                }
+
+                if (value.Length == 2)
+                {
+                    yield return Token.Name(text);
+                    yield break;
+                }
+
+                var first = text.Substring(0, 1);
+                yield return Token.Name(first);
+
+                var seen = new List<char> { first[0] };
+
+                foreach (var c in text.Substring(1))
+                {
+                    var n = new string(c, 1);
+                    if (!seen.Contains(c) && nameLookup(n))
+                    {
+                        seen.Add(c);
+                        yield return Token.Name(n);
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+                if (seen.Count() < text.Length)
+                {
+                    yield return Token.Value(text.Substring(seen.Count()));
+                }
+            }
+        }
+
+        private static IEnumerable<Token> TokenizeLongName(
+            string value,
+            Action<Error> onError)
+        {
+            if (value.Length > 2 && value.StartsWith("--", StringComparison.Ordinal))
+            {
+                var text = value.Substring(2);
+                var equalIndex = text.IndexOf('=');
+                if (equalIndex <= 0)
+                {
+                    yield return Token.Name(text);
+                    yield break;
+                }
+                if (equalIndex == 1) // "--="
+                {
+                    onError(new BadFormatTokenError(value));
+                    yield break;
+                }
+                var parts = text.Split('=');
+                yield return Token.Name(parts[0]);
+                yield return Token.Value(parts[1]);
+            }
+        }
+    }
 }
\ No newline at end of file
diff --git a/src/CommandLine/Core/TokenizerHybrid.cs b/src/CommandLine/Core/TokenizerHybrid.cs
new file mode 100644
index 00000000..a8508f60
--- /dev/null
+++ b/src/CommandLine/Core/TokenizerHybrid.cs
@@ -0,0 +1,143 @@
+// Copyright 2005-2015 Giacomo Stelluti Scala & Contributors. All rights reserved. See doc/License.md in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using CommandLine.Infrastructure;
+using CSharpx;
+using RailwaySharp.ErrorHandling;
+
+namespace CommandLine.Core
+{
+    internal class TokenizerHybrid : ITokenizer
+    {
+        public Result<IEnumerable<Token>, Error> Tokenize(
+            IEnumerable<string> arguments,
+            Func<string, bool> nameLookup)
+        {
+            var errors = new List<Error>();
+            Action<Error> onError = errors.Add;
+
+            var tokens = (from arg in arguments
+                          from token in !(arg.StartsWith("-", StringComparison.Ordinal) || arg.StartsWith("/", StringComparison.Ordinal))
+                               ? new[] { Token.Value(arg) }
+                               : (arg.StartsWith("--", StringComparison.Ordinal) || arg.StartsWith("//", StringComparison.Ordinal))
+                                     ? TokenizeLongName(arg, onError)
+                                     : TokenizeShortName(arg, nameLookup)
+                          select token).Memorize();
+
+            var unkTokens = (from t in tokens where t.IsName() && !nameLookup(t.Text) select t).Memorize();
+
+            return Result.Succeed(tokens.Where(x => !unkTokens.Contains(x)), errors.Concat(from t in unkTokens select new UnknownOptionError(t.Text)));
+        }
+
+        public Result<IEnumerable<Token>, Error> Preprocess(
+            IEnumerable<string> arguments,
+            Func<IEnumerable<string>, Result<IEnumerable<Token>, Error>> tokenizer)
+        {
+            if (arguments.Any(arg => arg.EqualsOrdinal("--")))
+            {
+                var tokenizerResult = tokenizer(arguments.TakeWhile(arg => !arg.EqualsOrdinal("--")));
+                var values = arguments.SkipWhile(arg => !arg.EqualsOrdinal("--")).Skip(1).Select(Token.Value);
+                return tokenizerResult.Map(tokens => tokens.Concat(values));
+            }
+            if (arguments.Any(arg => arg.EqualsOrdinal("//")))
+            {
+                var tokenizerResult = tokenizer(arguments.TakeWhile(arg => !arg.EqualsOrdinal("--")));
+                var values = arguments.SkipWhile(arg => !arg.EqualsOrdinal("//")).Skip(1).Select(Token.Value);
+                return tokenizerResult.Map(tokens => tokens.Concat(values));
+            }
+            return tokenizer(arguments);
+        }
+
+        public Result<IEnumerable<Token>, Error> ExplodeOptionList(
+            Result<IEnumerable<Token>, Error> tokenizerResult,
+            Func<string, Maybe<char>> optionSequenceWithSeparatorLookup)
+        {
+            var tokens = tokenizerResult.SucceededWith();
+
+            var replaces = tokens.Select((t,i) =>
+                optionSequenceWithSeparatorLookup(t.Text)
+                    .Return(sep => Tuple.Create(i + 1, sep),
+                        Tuple.Create(-1, '\0'))).SkipWhile(x => x.Item1 < 0);
+
+            var exploded = tokens.Select((t, i) =>
+                        replaces.FirstOrDefault(x => x.Item1 == i).ToMaybe()
+                            .Return(r => t.Text.Split(r.Item2).Select(Token.Value),
+                                Enumerable.Empty<Token>().Concat(new[]{ t })));
+
+            var flattened = exploded.SelectMany(x => x);
+
+            return Result.Succeed(flattened, tokenizerResult.SuccessfulMessages());
+        }
+
+        private static IEnumerable<Token> TokenizeShortName(
+            string value,
+            Func<string, bool> nameLookup)
+        {
+            if (value.Length > 1 || value[0] == '-' || value[1] != '-' || value [0] == '/' || value[1] != '/')
+            {
+                var text = value.Substring(1);
+
+                if (char.IsDigit(text[0]))
+                {
+                    yield return Token.Value(value);
+                    yield break;
+                }
+
+                if (value.Length == 2)
+                {
+                    yield return Token.Name(text);
+                    yield break;
+                }
+
+                var first = text.Substring(0, 1);
+                yield return Token.Name(first);
+
+                var seen = new List<char> { first[0] };
+
+                foreach (var c in text.Substring(1))
+                {
+                    var n = new string(c, 1);
+                    if (!seen.Contains(c) && nameLookup(n))
+                    {
+                        seen.Add(c);
+                        yield return Token.Name(n);
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+                if (seen.Count() < text.Length)
+                {
+                    yield return Token.Value(text.Substring(seen.Count()));
+                }
+            }
+        }
+
+        private static IEnumerable<Token> TokenizeLongName(
+            string value,
+            Action<Error> onError)
+        {
+            if (value.Length > 2 && (value.StartsWith("--", StringComparison.Ordinal) || value.StartsWith("//", StringComparison.Ordinal)))
+            {
+                var text = value.Substring(2);
+                var equalIndex = text.IndexOf('=');
+                if (equalIndex <= 0)
+                {
+                    yield return Token.Name(text);
+                    yield break;
+                }
+                if (equalIndex == 1) // "--="
+                {
+                    onError(new BadFormatTokenError(value));
+                    yield break;
+                }
+                var parts = text.Split('=');
+                yield return Token.Name(parts[0]);
+                yield return Token.Value(parts[1]);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/CommandLine/Core/TokenizerWindows.cs b/src/CommandLine/Core/TokenizerWindows.cs
new file mode 100644
index 00000000..f2c17c57
--- /dev/null
+++ b/src/CommandLine/Core/TokenizerWindows.cs
@@ -0,0 +1,137 @@
+// Copyright 2005-2015 Giacomo Stelluti Scala & Contributors. All rights reserved. See doc/License.md in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using CommandLine.Infrastructure;
+using CSharpx;
+using RailwaySharp.ErrorHandling;
+
+namespace CommandLine.Core
+{
+    internal class TokenizerWindows : ITokenizer
+    {
+        public Result<IEnumerable<Token>, Error> Tokenize(
+            IEnumerable<string> arguments,
+            Func<string, bool> nameLookup)
+        {
+            var errors = new List<Error>();
+            Action<Error> onError = errors.Add;
+
+            var tokens = (from arg in arguments
+                          from token in !arg.StartsWith("/", StringComparison.Ordinal)
+                               ? new[] { Token.Value(arg) }
+                               : arg.StartsWith("//", StringComparison.Ordinal)
+                                     ? TokenizeLongName(arg, onError)
+                                     : TokenizeShortName(arg, nameLookup)
+                          select token).Memorize();
+
+            var unkTokens = (from t in tokens where t.IsName() && !nameLookup(t.Text) select t).Memorize();
+
+            return Result.Succeed(tokens.Where(x => !unkTokens.Contains(x)), errors.Concat(from t in unkTokens select new UnknownOptionError(t.Text)));
+        }
+
+        public Result<IEnumerable<Token>, Error> Preprocess(
+            IEnumerable<string> arguments,
+            Func<IEnumerable<string>, Result<IEnumerable<Token>, Error>> tokenizer)
+        {
+            if (arguments.Any(arg => arg.EqualsOrdinal("//")))
+            {
+                var tokenizerResult = tokenizer(arguments.TakeWhile(arg => !arg.EqualsOrdinal("--")));
+                var values = arguments.SkipWhile(arg => !arg.EqualsOrdinal("//")).Skip(1).Select(Token.Value);
+                return tokenizerResult.Map(tokens => tokens.Concat(values));
+            }
+            return tokenizer(arguments);
+        }
+
+        public Result<IEnumerable<Token>, Error> ExplodeOptionList(
+            Result<IEnumerable<Token>, Error> tokenizerResult,
+            Func<string, Maybe<char>> optionSequenceWithSeparatorLookup)
+        {
+            var tokens = tokenizerResult.SucceededWith();
+
+            var replaces = tokens.Select((t,i) =>
+                optionSequenceWithSeparatorLookup(t.Text)
+                    .Return(sep => Tuple.Create(i + 1, sep),
+                        Tuple.Create(-1, '\0'))).SkipWhile(x => x.Item1 < 0);
+
+            var exploded = tokens.Select((t, i) =>
+                        replaces.FirstOrDefault(x => x.Item1 == i).ToMaybe()
+                            .Return(r => t.Text.Split(r.Item2).Select(Token.Value),
+                                Enumerable.Empty<Token>().Concat(new[]{ t })));
+
+            var flattened = exploded.SelectMany(x => x);
+
+            return Result.Succeed(flattened, tokenizerResult.SuccessfulMessages());
+        }
+
+        private IEnumerable<Token> TokenizeShortName(
+            string value,
+            Func<string, bool> nameLookup)
+        {
+            if (value.Length > 1 || value[0] == '/' || value[1] != '/')
+            {
+                var text = value.Substring(1);
+
+                if (char.IsDigit(text[0]))
+                {
+                    yield return Token.Value(value);
+                    yield break;
+                }
+
+                if (value.Length == 2)
+                {
+                    yield return Token.Name(text);
+                    yield break;
+                }
+
+                var first = text.Substring(0, 1);
+                yield return Token.Name(first);
+
+                var seen = new List<char> { first[0] };
+
+                foreach (var c in text.Substring(1))
+                {
+                    var n = new string(c, 1);
+                    if (!seen.Contains(c) && nameLookup(n))
+                    {
+                        seen.Add(c);
+                        yield return Token.Name(n);
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+                if (seen.Count() < text.Length)
+                {
+                    yield return Token.Value(text.Substring(seen.Count()));
+                }
+            }
+        }
+
+        private IEnumerable<Token> TokenizeLongName(
+            string value,
+            Action<Error> onError)
+        {
+            if (value.Length > 2 && value.StartsWith("//", StringComparison.Ordinal))
+            {
+                var text = value.Substring(2);
+                var equalIndex = text.IndexOf('=');
+                if (equalIndex <= 0)
+                {
+                    yield return Token.Name(text);
+                    yield break;
+                }
+                if (equalIndex == 1) // "--="
+                {
+                    onError(new BadFormatTokenError(value));
+                    yield break;
+                }
+                var parts = text.Split('=');
+                yield return Token.Name(parts[0]);
+                yield return Token.Value(parts[1]);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/CommandLine/Parser.cs b/src/CommandLine/Parser.cs
index 1bd04d3b..42c829d4 100644
--- a/src/CommandLine/Parser.cs
+++ b/src/CommandLine/Parser.cs
@@ -171,13 +171,15 @@ private static Result<IEnumerable<Token>, Error> Tokenize(
                 IEnumerable<OptionSpecification> optionSpecs,
                 ParserSettings settings)
         {
+            var tokenizer = new TokenizerGetOpt();
+
             var tokens = settings.EnableDashDash
-                ? Tokenizer.PreprocessDashDash(
+                ? tokenizer.Preprocess(
                         arguments,
                         args =>
-                            Tokenizer.Tokenize(args, name => NameLookup.Contains(name, optionSpecs, settings.NameComparer)))
-                : Tokenizer.Tokenize(arguments, name => NameLookup.Contains(name, optionSpecs, settings.NameComparer));
-            var explodedTokens = Tokenizer.ExplodeOptionList(tokens, name => NameLookup.HavingSeparator(name, optionSpecs, settings.NameComparer));
+                            tokenizer.Tokenize(args, name => NameLookup.Contains(name, optionSpecs, settings.NameComparer)))
+                : tokenizer.Tokenize(arguments, name => NameLookup.Contains(name, optionSpecs, settings.NameComparer));
+            var explodedTokens = tokenizer.ExplodeOptionList(tokens, name => NameLookup.HavingSeparator(name, optionSpecs, settings.NameComparer));
             return explodedTokens;
         }