From a279f5300e8ff37670536757a5a824c583564b2b Mon Sep 17 00:00:00 2001 From: vxern Date: Tue, 31 Oct 2023 18:40:31 +0000 Subject: [PATCH] feat: Bump versions, remove the `meta` package, resolve remaining TODO. --- CHANGELOG.md | 7 +++++++ README.md | 6 +++--- example/example.dart | 4 ++-- lib/robots_txt.dart | 6 +++--- lib/src/robots.dart | 21 --------------------- lib/src/rule.dart | 5 ----- lib/src/ruleset.dart | 4 ---- pubspec.yaml | 11 ++++------- 8 files changed, 19 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d94283a..19475ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 2.3.0 + +- Bump dependencies. +- Bump the SDK version to `3.0.0`. +- Remove the `meta` package. +- Remove work-around for issue fixed by https://github.com/dart-lang/sdk/issues/49188. + ## 2.2.0+3 - Updated licence bearer. diff --git a/README.md b/README.md index 1827189..52240fe 100644 --- a/README.md +++ b/README.md @@ -17,15 +17,15 @@ establish whether or not a user-agent is allowed to visit a particular path: ```dart final userAgent = /* Your user-agent. */; print(robots.verifyCanAccess('/gist/', userAgent: userAgent)); // False -print(robots.verifyCanAccess('/wordcollector/robots_txt/', userAgent: userAgent)); // True +print(robots.verifyCanAccess('/government/robots_txt/', userAgent: userAgent)); // True ``` If you are only concerned about directives pertaining to your own user-agent, you may instruct the parser to ignore other user-agents as follows: ```dart -// Parse the contents, disregarding user-agents other than 'WordCollector'. -final robots = Robots.parse(contents, onlyApplicableTo: const {'WordCollector'}); +// Parse the contents, disregarding user-agents other than 'government'. +final robots = Robots.parse(contents, onlyApplicableTo: const {'government'}); ``` The `Robots.parse()` function does not have any built-in structure validation. diff --git a/example/example.dart b/example/example.dart index 8b1827f..710154d 100644 --- a/example/example.dart +++ b/example/example.dart @@ -33,7 +33,7 @@ Future main() async { } } - const userAgent = 'WordCollector'; + const userAgent = 'government'; // False: it cannot. print( @@ -41,7 +41,7 @@ Future main() async { ); // True: it can. print( - "Can '$userAgent' access /wordcollector/robots_txt/? ${robots.verifyCanAccess('/wordcollector/robots_txt/', userAgent: userAgent)}", + "Can '$userAgent' access /government/robots_txt/? ${robots.verifyCanAccess('/government/robots_txt/', userAgent: userAgent)}", ); // Validating an invalid file will throw a `FormatException`. diff --git a/lib/robots_txt.dart b/lib/robots_txt.dart index f5a9159..d0d8543 100644 --- a/lib/robots_txt.dart +++ b/lib/robots_txt.dart @@ -1,6 +1,6 @@ /// Lightweight, fully documented `robots.txt` file parser. library robots_txt; -export 'src/robots.dart' show Robots, PrecedentRuleType, FieldType; -export 'src/rule.dart' show Rule, FindRule, Precedence, PrecedenceStrategy; -export 'src/ruleset.dart' show Ruleset, FindRuleInRuleset; +export 'src/robots.dart' show FieldType, PrecedentRuleType, Robots; +export 'src/rule.dart' show FindRule, Precedence, PrecedenceStrategy, Rule; +export 'src/ruleset.dart' show FindRuleInRuleset, Ruleset; diff --git a/lib/src/robots.dart b/lib/src/robots.dart index 85c07d0..63d8ccf 100644 --- a/lib/src/robots.dart +++ b/lib/src/robots.dart @@ -1,11 +1,8 @@ -import 'package:meta/meta.dart'; - import 'package:robots_txt/src/rule.dart'; import 'package:robots_txt/src/ruleset.dart'; /// Taking a set of [allowedFieldNames], builds a regular expression matching /// only to valid `robots.txt` files. -@internal RegExp buildValidFilePattern({Set allowedFieldNames = const {}}) { final fieldNameExpression = [FieldType.defaultFieldNameExpression, ...allowedFieldNames].join('|'); @@ -16,14 +13,11 @@ RegExp buildValidFilePattern({Set allowedFieldNames = const {}}) { } /// Defines a Regex pattern that matches to comments. -@internal final commentPattern = RegExp('#.*'); /// Stores information about a `robots.txt` file, exposing a simple and concise /// API for working with the file and validating if a certain path can be /// accessed by a given user-agent. -@immutable -@sealed class Robots { /// Stores information about the rules specified for given user-agents. final List rulesets; @@ -168,7 +162,6 @@ class Robots { } userAgents.add(field.value); - break; case FieldType.disallow: if (!isReadingRuleset()) { break; @@ -190,8 +183,6 @@ class Robots { precedence: lines.length - (index + 1), ), ); - - break; case FieldType.allow: if (!isReadingRuleset()) { break; @@ -213,11 +204,8 @@ class Robots { precedence: lines.length - (index + 1), ), ); - - break; case FieldType.sitemap: sitemaps.add(field.value); - break; case FieldType.crawlDelay: final value = int.tryParse(field.value); if (value == null || (crawlDelay != null && value < crawlDelay!)) { @@ -225,10 +213,8 @@ class Robots { } crawlDelay = value; - break; case FieldType.host: hosts.add(field.value); - break; } previousType = type; @@ -296,10 +282,6 @@ class Robots { ); switch (typePrecedence) { - case PrecedentRuleType.defaultPrecedentType: - // TODO(vxern): Below is a fix for an issue in Dart 2.18 with the enhanced - // enums. This issue is fixed in 2.19, which is still on the beta - // channel. Refer to: https://github.com/dart-lang/sdk/issues/49188 // ignore: no_duplicate_case_values case PrecedentRuleType.allow: return allowedBy != null || disallowedBy == null; @@ -311,7 +293,6 @@ class Robots { /// Taking an [exception], a [line] and the [index] of that line, creates a more /// informational `FormatException`. -@internal FormatException wrapFormatException( Exception exception, String line, @@ -325,7 +306,6 @@ is invalid: '''); /// Describes the type of a rule. -@internal enum RuleType { /// A rule explicitly allows a given path. allow, @@ -351,7 +331,6 @@ enum PrecedentRuleType { } /// Defines a key-value field of a `robots.txt` file specifying a rule. -@visibleForTesting enum FieldType { /// A field specifying the user-agent the following fields apply to. userAgent(key: 'User-agent', example: '*'), diff --git a/lib/src/rule.dart b/lib/src/rule.dart index 86bc0cd..0874767 100644 --- a/lib/src/rule.dart +++ b/lib/src/rule.dart @@ -1,8 +1,4 @@ -import 'package:meta/meta.dart'; - /// A single rule (either `Allow` or `Disallow`) inside a `robots.txt` file. -@immutable -@sealed class Rule { /// A regular expression matching to a particular path. final RegExp pattern; @@ -52,7 +48,6 @@ extension Precedence on Rule? { /// The signature of a method that compares two variables of type `T` and /// returns the one supposed 'greater'. -@internal typedef ComparisonFunction = T Function(T a, T b); /// `ComparisonFunction`s matched to `PrecedenceStrategy`s. diff --git a/lib/src/ruleset.dart b/lib/src/ruleset.dart index 7ab66b7..c01f01a 100644 --- a/lib/src/ruleset.dart +++ b/lib/src/ruleset.dart @@ -1,11 +1,7 @@ -import 'package:meta/meta.dart'; - import 'package:robots_txt/src/robots.dart'; import 'package:robots_txt/src/rule.dart'; /// A collection of `Rule`s applicable to a particular [userAgent]. -@immutable -@sealed class Ruleset { /// The user-agent which this ruleset applies to. final String userAgent; diff --git a/pubspec.yaml b/pubspec.yaml index 1c243c8..80b2c7f 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,5 +1,5 @@ name: robots_txt -version: 2.2.0+3 +version: 2.3.0 description: A complete, dependency-less and fully documented `robots.txt` ruleset parser. @@ -8,11 +8,8 @@ repository: https://github.com/vxern/robots_txt issue_tracker: https://github.com/vxern/robots_txt/issues environment: - sdk: '>=2.17.0 <3.0.0' - -dependencies: - meta: ^1.8.0 # Used for static analysis. + sdk: ">=3.0.0 <4.0.0" dev_dependencies: - test: ^1.22.1 # Testing. - words: ^0.2.0 # Stricter lints. + test: ^1.24.9 # Testing. + words: ^0.4.4 # Stricter lints.