Skip to content

Commit

Permalink
feat: Bump versions, remove the meta package, resolve remaining TODO.
Browse files Browse the repository at this point in the history
  • Loading branch information
vxern committed Oct 31, 2023
1 parent 2b967da commit a279f53
Show file tree
Hide file tree
Showing 8 changed files with 19 additions and 45 deletions.
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
## 2.3.0

- Bump dependencies.
- Bump the SDK version to `3.0.0`.
- Remove the `meta` package.
- Remove work-around for issue fixed by https://github.com/dart-lang/sdk/issues/49188.

## 2.2.0+3

- Updated licence bearer.
Expand Down
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@ establish whether or not a user-agent is allowed to visit a particular path:
```dart
final userAgent = /* Your user-agent. */;
print(robots.verifyCanAccess('/gist/', userAgent: userAgent)); // False
print(robots.verifyCanAccess('/wordcollector/robots_txt/', userAgent: userAgent)); // True
print(robots.verifyCanAccess('/government/robots_txt/', userAgent: userAgent)); // True
```

If you are only concerned about directives pertaining to your own user-agent,
you may instruct the parser to ignore other user-agents as follows:

```dart
// Parse the contents, disregarding user-agents other than 'WordCollector'.
final robots = Robots.parse(contents, onlyApplicableTo: const {'WordCollector'});
// Parse the contents, disregarding user-agents other than 'government'.
final robots = Robots.parse(contents, onlyApplicableTo: const {'government'});
```

The `Robots.parse()` function does not have any built-in structure validation.
Expand Down
4 changes: 2 additions & 2 deletions example/example.dart
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,15 @@ Future<void> main() async {
}
}

const userAgent = 'WordCollector';
const userAgent = 'government';

// False: it cannot.
print(
"Can '$userAgent' access /gist/? ${robots.verifyCanAccess('/gist/', userAgent: userAgent)}",
);
// True: it can.
print(
"Can '$userAgent' access /wordcollector/robots_txt/? ${robots.verifyCanAccess('/wordcollector/robots_txt/', userAgent: userAgent)}",
"Can '$userAgent' access /government/robots_txt/? ${robots.verifyCanAccess('/government/robots_txt/', userAgent: userAgent)}",
);

// Validating an invalid file will throw a `FormatException`.
Expand Down
6 changes: 3 additions & 3 deletions lib/robots_txt.dart
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/// Lightweight, fully documented `robots.txt` file parser.
library robots_txt;

export 'src/robots.dart' show Robots, PrecedentRuleType, FieldType;
export 'src/rule.dart' show Rule, FindRule, Precedence, PrecedenceStrategy;
export 'src/ruleset.dart' show Ruleset, FindRuleInRuleset;
export 'src/robots.dart' show FieldType, PrecedentRuleType, Robots;
export 'src/rule.dart' show FindRule, Precedence, PrecedenceStrategy, Rule;
export 'src/ruleset.dart' show FindRuleInRuleset, Ruleset;
21 changes: 0 additions & 21 deletions lib/src/robots.dart
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
import 'package:meta/meta.dart';

import 'package:robots_txt/src/rule.dart';
import 'package:robots_txt/src/ruleset.dart';

/// Taking a set of [allowedFieldNames], builds a regular expression matching
/// only to valid `robots.txt` files.
@internal
RegExp buildValidFilePattern({Set<String> allowedFieldNames = const {}}) {
final fieldNameExpression =
[FieldType.defaultFieldNameExpression, ...allowedFieldNames].join('|');
Expand All @@ -16,14 +13,11 @@ RegExp buildValidFilePattern({Set<String> allowedFieldNames = const {}}) {
}

/// Defines a Regex pattern that matches to comments.
@internal
final commentPattern = RegExp('#.*');

/// Stores information about a `robots.txt` file, exposing a simple and concise
/// API for working with the file and validating if a certain path can be
/// accessed by a given user-agent.
@immutable
@sealed
class Robots {
/// Stores information about the rules specified for given user-agents.
final List<Ruleset> rulesets;
Expand Down Expand Up @@ -168,7 +162,6 @@ class Robots {
}

userAgents.add(field.value);
break;
case FieldType.disallow:
if (!isReadingRuleset()) {
break;
Expand All @@ -190,8 +183,6 @@ class Robots {
precedence: lines.length - (index + 1),
),
);

break;
case FieldType.allow:
if (!isReadingRuleset()) {
break;
Expand All @@ -213,22 +204,17 @@ class Robots {
precedence: lines.length - (index + 1),
),
);

break;
case FieldType.sitemap:
sitemaps.add(field.value);
break;
case FieldType.crawlDelay:
final value = int.tryParse(field.value);
if (value == null || (crawlDelay != null && value < crawlDelay!)) {
break;
}

crawlDelay = value;
break;
case FieldType.host:
hosts.add(field.value);
break;
}

previousType = type;
Expand Down Expand Up @@ -296,10 +282,6 @@ class Robots {
);

switch (typePrecedence) {
case PrecedentRuleType.defaultPrecedentType:
// TODO(vxern): Below is a fix for an issue in Dart 2.18 with the enhanced
// enums. This issue is fixed in 2.19, which is still on the beta
// channel. Refer to: https://github.com/dart-lang/sdk/issues/49188
// ignore: no_duplicate_case_values
case PrecedentRuleType.allow:
return allowedBy != null || disallowedBy == null;
Expand All @@ -311,7 +293,6 @@ class Robots {

/// Taking an [exception], a [line] and the [index] of that line, creates a more
/// informational `FormatException`.
@internal
FormatException wrapFormatException(
Exception exception,
String line,
Expand All @@ -325,7 +306,6 @@ is invalid:
''');

/// Describes the type of a rule.
@internal
enum RuleType {
/// A rule explicitly allows a given path.
allow,
Expand All @@ -351,7 +331,6 @@ enum PrecedentRuleType {
}

/// Defines a key-value field of a `robots.txt` file specifying a rule.
@visibleForTesting
enum FieldType {
/// A field specifying the user-agent the following fields apply to.
userAgent(key: 'User-agent', example: '*'),
Expand Down
5 changes: 0 additions & 5 deletions lib/src/rule.dart
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
import 'package:meta/meta.dart';

/// A single rule (either `Allow` or `Disallow`) inside a `robots.txt` file.
@immutable
@sealed
class Rule {
/// A regular expression matching to a particular path.
final RegExp pattern;
Expand Down Expand Up @@ -52,7 +48,6 @@ extension Precedence on Rule? {

/// The signature of a method that compares two variables of type `T` and
/// returns the one supposed 'greater'.
@internal
typedef ComparisonFunction<T> = T Function(T a, T b);

/// `ComparisonFunction`s matched to `PrecedenceStrategy`s.
Expand Down
4 changes: 0 additions & 4 deletions lib/src/ruleset.dart
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
import 'package:meta/meta.dart';

import 'package:robots_txt/src/robots.dart';
import 'package:robots_txt/src/rule.dart';

/// A collection of `Rule`s applicable to a particular [userAgent].
@immutable
@sealed
class Ruleset {
/// The user-agent which this ruleset applies to.
final String userAgent;
Expand Down
11 changes: 4 additions & 7 deletions pubspec.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: robots_txt
version: 2.2.0+3
version: 2.3.0

description: A complete, dependency-less and fully documented `robots.txt` ruleset parser.

Expand All @@ -8,11 +8,8 @@ repository: https://github.com/vxern/robots_txt
issue_tracker: https://github.com/vxern/robots_txt/issues

environment:
sdk: '>=2.17.0 <3.0.0'

dependencies:
meta: ^1.8.0 # Used for static analysis.
sdk: ">=3.0.0 <4.0.0"

dev_dependencies:
test: ^1.22.1 # Testing.
words: ^0.2.0 # Stricter lints.
test: ^1.24.9 # Testing.
words: ^0.4.4 # Stricter lints.

0 comments on commit a279f53

Please sign in to comment.