Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added multiple tokenizers implementing new ITokenizer interface #212

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Added multiple tokenizers implementing new ITokenizer interface #212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e69798d
Added multiple tokenizers implementing new ITokenizer interface (GetO…
Ehryk Aug 1, 2015
e444e69
Resolving Merge Conflicts
Ehryk Aug 2, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/CommandLine/CommandLine.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@
<Compile Include="Core\ArgumentsExtensions.cs" />
<Compile Include="Core\KeyValuePairHelper.cs" />
<Compile Include="Core\PreprocessorGuards.cs" />
<Compile Include="Core\ITokenizer.cs" />
<Compile Include="Core\TokenizerHybrid.cs" />
<Compile Include="Core\TokenizerWindows.cs" />
<Compile Include="Core\TypeDescriptor.cs" />
<Compile Include="Core\Scalar.cs" />
<Compile Include="Core\Sequence.cs" />
Expand All @@ -82,7 +85,7 @@
<Compile Include="ErrorExtensions.cs" />
<Compile Include="Infrastructure\EnumerableExtensions`1.cs" />
<Compile Include="Core\InstanceChooser.cs" />
<Compile Include="Core\Tokenizer.cs">
<Compile Include="Core\TokenizerGetOpt.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="Core\Verb.cs" />
Expand Down Expand Up @@ -148,6 +151,7 @@
<Compile Include="ValueAttribute.cs" />
<Compile Include="VerbAttribute.cs" />
</ItemGroup>
<ItemGroup />
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
Expand Down
26 changes: 26 additions & 0 deletions src/CommandLine/Core/ITokenizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// Copyright 2005-2015 Giacomo Stelluti Scala & Contributors. All rights reserved. See doc/License.md in the project root for license information.

using System;
using System.Collections.Generic;
using System.Linq;
using CommandLine.Infrastructure;
using CSharpx;
using RailwaySharp.ErrorHandling;

namespace CommandLine.Core
{
internal interface ITokenizer
{
Result<IEnumerable<Token>, Error> Tokenize(
IEnumerable<string> arguments,
Func<string, bool> nameLookup);

Result<IEnumerable<Token>, Error> Preprocess(
IEnumerable<string> arguments,
Func<IEnumerable<string>, Result<IEnumerable<Token>, Error>> tokenizer);

Result<IEnumerable<Token>, Error> ExplodeOptionList(
Result<IEnumerable<Token>, Error> tokenizerResult,
Func<string, Maybe<char>> optionSequenceWithSeparatorLookup);
}
}
6 changes: 4 additions & 2 deletions src/CommandLine/Core/InstanceBuilder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@ public static ParserResult<T> Build<T>(
StringComparer nameComparer,
CultureInfo parsingCulture)
{
var tokenizer = new TokenizerGetOpt();

return Build(
factory,
(args, optionSpecs) =>
{
var tokens = Tokenizer.Tokenize(args, name => NameLookup.Contains(name, optionSpecs, nameComparer));
var explodedTokens = Tokenizer.ExplodeOptionList(
var tokens = tokenizer.Tokenize(args, name => NameLookup.Contains(name, optionSpecs, nameComparer));
var explodedTokens = tokenizer.ExplodeOptionList(
tokens,
name => NameLookup.HavingSeparator(name, optionSpecs, nameComparer));
return explodedTokens;
Expand Down
6 changes: 4 additions & 2 deletions src/CommandLine/Core/InstanceChooser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@ public static ParserResult<object> Choose(
StringComparer nameComparer,
CultureInfo parsingCulture)
{
var tokenizer = new TokenizerGetOpt();

return Choose(
(args, optionSpecs) =>
{
var tokens = Tokenizer.Tokenize(args, name => NameLookup.Contains(name, optionSpecs, nameComparer));
var explodedTokens = Tokenizer.ExplodeOptionList(tokens, name => NameLookup.HavingSeparator(name, optionSpecs, nameComparer));
var tokens = tokenizer.Tokenize(args, name => NameLookup.Contains(name, optionSpecs, nameComparer));
var explodedTokens = tokenizer.ExplodeOptionList(tokens, name => NameLookup.HavingSeparator(name, optionSpecs, nameComparer));
return explodedTokens;
},
types,
Expand Down
272 changes: 136 additions & 136 deletions src/CommandLine/Core/Tokenizer.cs → src/CommandLine/Core/TokenizerGetOpt.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,137 +1,137 @@
// Copyright 2005-2015 Giacomo Stelluti Scala & Contributors. All rights reserved. See License.md in the project root for license information.
using System;
using System.Collections.Generic;
using System.Linq;
using CommandLine.Infrastructure;
using CSharpx;
using RailwaySharp.ErrorHandling;
namespace CommandLine.Core
{
internal static class Tokenizer
{
public static Result<IEnumerable<Token>, Error> Tokenize(
IEnumerable<string> arguments,
Func<string, bool> nameLookup)
{
var errors = new List<Error>();
Action<Error> onError = errors.Add;
var tokens = (from arg in arguments
from token in !arg.StartsWith("-", StringComparison.Ordinal)
? new[] { Token.Value(arg) }
: arg.StartsWith("--", StringComparison.Ordinal)
? TokenizeLongName(arg, onError)
: TokenizeShortName(arg, nameLookup)
select token).Memorize();
var unkTokens = (from t in tokens where t.IsName() && !nameLookup(t.Text) select t).Memorize();
return Result.Succeed(tokens.Where(x => !unkTokens.Contains(x)), errors.Concat(from t in unkTokens select new UnknownOptionError(t.Text)));
}
public static Result<IEnumerable<Token>, Error> PreprocessDashDash(
IEnumerable<string> arguments,
Func<IEnumerable<string>, Result<IEnumerable<Token>, Error>> tokenizer)
{
if (arguments.Any(arg => arg.EqualsOrdinal("--")))
{
var tokenizerResult = tokenizer(arguments.TakeWhile(arg => !arg.EqualsOrdinal("--")));
var values = arguments.SkipWhile(arg => !arg.EqualsOrdinal("--")).Skip(1).Select(Token.Value);
return tokenizerResult.Map(tokens => tokens.Concat(values));
}
return tokenizer(arguments);
}
public static Result<IEnumerable<Token>, Error> ExplodeOptionList(
Result<IEnumerable<Token>, Error> tokenizerResult,
Func<string, Maybe<char>> optionSequenceWithSeparatorLookup)
{
var tokens = tokenizerResult.SucceededWith();
var replaces = tokens.Select((t,i) =>
optionSequenceWithSeparatorLookup(t.Text)
.Return(sep => Tuple.Create(i + 1, sep),
Tuple.Create(-1, '\0'))).SkipWhile(x => x.Item1 < 0);
var exploded = tokens.Select((t, i) =>
replaces.FirstOrDefault(x => x.Item1 == i).ToMaybe()
.Return(r => t.Text.Split(r.Item2).Select(Token.Value),
Enumerable.Empty<Token>().Concat(new[]{ t })));
var flattened = exploded.SelectMany(x => x);
return Result.Succeed(flattened, tokenizerResult.SuccessfulMessages());
}
private static IEnumerable<Token> TokenizeShortName(
string value,
Func<string, bool> nameLookup)
{
if (value.Length > 1 || value[0] == '-' || value[1] != '-')
{
var text = value.Substring(1);
if (char.IsDigit(text[0]))
{
yield return Token.Value(value);
yield break;
}
if (value.Length == 2)
{
yield return Token.Name(text);
yield break;
}
var first = text.Substring(0, 1);
yield return Token.Name(first);
var seen = new List<char> { first[0] };
foreach (var c in text.Substring(1))
{
var n = new string(c, 1);
if (!seen.Contains(c) && nameLookup(n))
{
seen.Add(c);
yield return Token.Name(n);
}
else
{
break;
}
}
if (seen.Count() < text.Length)
{
yield return Token.Value(text.Substring(seen.Count()));
}
}
}
private static IEnumerable<Token> TokenizeLongName(
string value,
Action<Error> onError)
{
if (value.Length > 2 && value.StartsWith("--", StringComparison.Ordinal))
{
var text = value.Substring(2);
var equalIndex = text.IndexOf('=');
if (equalIndex <= 0)
{
yield return Token.Name(text);
yield break;
}
if (equalIndex == 1) // "--="
{
onError(new BadFormatTokenError(value));
yield break;
}
var parts = text.Split('=');
yield return Token.Name(parts[0]);
yield return Token.Value(parts[1]);
}
}
}
// Copyright 2005-2015 Giacomo Stelluti Scala & Contributors. All rights reserved. See License.md in the project root for license information.

using System;
using System.Collections.Generic;
using System.Linq;
using CommandLine.Infrastructure;
using CSharpx;
using RailwaySharp.ErrorHandling;

namespace CommandLine.Core
{
internal class TokenizerGetOpt : ITokenizer
{
public Result<IEnumerable<Token>, Error> Tokenize(
IEnumerable<string> arguments,
Func<string, bool> nameLookup)
{
var errors = new List<Error>();
Action<Error> onError = errors.Add;

var tokens = (from arg in arguments
from token in !arg.StartsWith("-", StringComparison.Ordinal)
? new[] { Token.Value(arg) }
: arg.StartsWith("--", StringComparison.Ordinal)
? TokenizeLongName(arg, onError)
: TokenizeShortName(arg, nameLookup)
select token).Memorize();

var unkTokens = (from t in tokens where t.IsName() && !nameLookup(t.Text) select t).Memorize();

return Result.Succeed(tokens.Where(x => !unkTokens.Contains(x)), errors.Concat(from t in unkTokens select new UnknownOptionError(t.Text)));
}

public Result<IEnumerable<Token>, Error> Preprocess(
IEnumerable<string> arguments,
Func<IEnumerable<string>, Result<IEnumerable<Token>, Error>> tokenizer)
{
if (arguments.Any(arg => arg.EqualsOrdinal("--")))
{
var tokenizerResult = tokenizer(arguments.TakeWhile(arg => !arg.EqualsOrdinal("--")));
var values = arguments.SkipWhile(arg => !arg.EqualsOrdinal("--")).Skip(1).Select(Token.Value);
return tokenizerResult.Map(tokens => tokens.Concat(values));
}
return tokenizer(arguments);
}

public Result<IEnumerable<Token>, Error> ExplodeOptionList(
Result<IEnumerable<Token>, Error> tokenizerResult,
Func<string, Maybe<char>> optionSequenceWithSeparatorLookup)
{
var tokens = tokenizerResult.SucceededWith();

var replaces = tokens.Select((t,i) =>
optionSequenceWithSeparatorLookup(t.Text)
.Return(sep => Tuple.Create(i + 1, sep),
Tuple.Create(-1, '\0'))).SkipWhile(x => x.Item1 < 0);

var exploded = tokens.Select((t, i) =>
replaces.FirstOrDefault(x => x.Item1 == i).ToMaybe()
.Return(r => t.Text.Split(r.Item2).Select(Token.Value),
Enumerable.Empty<Token>().Concat(new[]{ t })));

var flattened = exploded.SelectMany(x => x);

return Result.Succeed(flattened, tokenizerResult.SuccessfulMessages());
}

private static IEnumerable<Token> TokenizeShortName(
string value,
Func<string, bool> nameLookup)
{
if (value.Length > 1 || value[0] == '-' || value[1] != '-')
{
var text = value.Substring(1);

if (char.IsDigit(text[0]))
{
yield return Token.Value(value);
yield break;
}

if (value.Length == 2)
{
yield return Token.Name(text);
yield break;
}

var first = text.Substring(0, 1);
yield return Token.Name(first);

var seen = new List<char> { first[0] };

foreach (var c in text.Substring(1))
{
var n = new string(c, 1);
if (!seen.Contains(c) && nameLookup(n))
{
seen.Add(c);
yield return Token.Name(n);
}
else
{
break;
}
}
if (seen.Count() < text.Length)
{
yield return Token.Value(text.Substring(seen.Count()));
}
}
}

private static IEnumerable<Token> TokenizeLongName(
string value,
Action<Error> onError)
{
if (value.Length > 2 && value.StartsWith("--", StringComparison.Ordinal))
{
var text = value.Substring(2);
var equalIndex = text.IndexOf('=');
if (equalIndex <= 0)
{
yield return Token.Name(text);
yield break;
}
if (equalIndex == 1) // "--="
{
onError(new BadFormatTokenError(value));
yield break;
}
var parts = text.Split('=');
yield return Token.Name(parts[0]);
yield return Token.Value(parts[1]);
}
}
}
}
Loading