Allow specifying an explicit location for test/groups (#2481)

Usually test locations (which are used both to report the location of
tests in the json reporter, and also for filtering tests by line/col)
are inferred from the call stack when test() is called.

This changes adds a new `location` parameter to group/test to allow this
information to be provided explicitly. This can be useful where the call
to `test()` doesn't contain the actual location of the declaration in
the call stack (for example when using `pkg:test_reflective_loader`).

See #2029
See Dart-Code/Dart-Code#5480

Author: DanTup

.gitignore

@@ -1,6 +1,7 @@
 # Don’t commit the following directories created by pub.
 .buildlog
 .dart_tool/
+.vscode/settings.json
 .pub/
 build/
 packages

pkgs/test/CHANGELOG.md

@@ -1,4 +1,8 @@
-## 1.25.16-wip
+## 1.26.0-wip
+
+* `test()` and `group()` functions now take an optional `TestLocation` that will
+  be used as the location of the test in JSON reporters instead of being parsed
+  from the call stack.
 
 ## 1.25.15
 

pkgs/test/pubspec.yaml

@@ -1,5 +1,5 @@
 name: test
-version: 1.25.16-wip
+version: 1.26.0-wip
 description: >-
   A full featured library for writing and running Dart tests across platforms.
 repository: https://github.com/dart-lang/test/tree/master/pkgs/test
@@ -36,7 +36,7 @@ dependencies:
   stream_channel: ^2.1.0
 
   # Use an exact version until the test_api and test_core package are stable.
-  test_api: 0.7.4
+  test_api: 0.7.5-wip
   test_core: 0.6.9-wip
 
   typed_data: ^1.3.0

pkgs/test/test/runner/engine_test.dart

@@ -7,6 +7,7 @@ import 'dart:math';
 
 import 'package:test/test.dart';
 import 'package:test_api/src/backend/group.dart';
+import 'package:test_api/src/backend/group_entry.dart';
 import 'package:test_api/src/backend/state.dart';
 import 'package:test_core/src/runner/engine.dart';
 
@@ -27,8 +28,8 @@ void main() {
     });
 
     var engine = Engine.withSuites([
-      runnerSuite(Group.root(tests.take(2))),
-      runnerSuite(Group.root(tests.skip(2)))
+      runnerSuite(tests.take(2).asRootGroup()),
+      runnerSuite(tests.skip(2).asRootGroup())
     ]);
 
     await engine.run();
@@ -55,7 +56,7 @@ void main() {
         }),
         completes);
 
-    engine.suiteSink.add(runnerSuite(Group.root(tests)));
+    engine.suiteSink.add(runnerSuite(tests.asRootGroup()));
     engine.suiteSink.close();
   });
 
@@ -207,7 +208,7 @@ void main() {
         test('test', () {}, skip: true);
       });
 
-      var engine = Engine.withSuites([runnerSuite(Group.root(tests))]);
+      var engine = Engine.withSuites([runnerSuite(tests.asRootGroup())]);
 
       engine.onTestStarted.listen(expectAsync1((liveTest) {
         expect(liveTest, same(engine.liveTests.single));
@@ -276,7 +277,7 @@ void main() {
         }, skip: true);
       });
 
-      var engine = Engine.withSuites([runnerSuite(Group.root(entries))]);
+      var engine = Engine.withSuites([runnerSuite(entries.asRootGroup())]);
 
       engine.onTestStarted.listen(expectAsync1((liveTest) {
         expect(liveTest, same(engine.liveTests.single));
@@ -341,7 +342,7 @@ void main() {
           for (var i = 0; i < testCount; i++)
             loadSuite('group $i', () async {
               await updateAndCheckConcurrency(isLoadSuite: true);
-              return runnerSuite(Group.root([tests[i]]));
+              return runnerSuite([tests[i]].asRootGroup());
             }),
         ], concurrency: concurrency);
 
@@ -355,3 +356,11 @@ void main() {
     });
   });
 }
+
+extension on Iterable<GroupEntry> {
+  /// Clones these entries into a new root group, assigning the new parent group
+  /// as necessary.
+  Group asRootGroup() {
+    return Group.root(map((entry) => entry.filter((_) => true)!));
+  }
+}

pkgs/test/test/runner/json_reporter_test.dart

@@ -587,6 +587,70 @@ void customTest(String name, dynamic Function() testFn) => test(name, testFn);
 ''',
           });
     });
+
+    test('groups and tests with custom locations', () {
+      return _expectReport('''
+      group('group 1 inferred', () {
+        setUpAll(() {});
+        test('test 1 inferred', () {});
+        tearDownAll(() {});
+      });
+      group('group 2 custom', location: TestLocation(Uri.parse('file:///foo/group'), 123, 234), () {
+        setUpAll(location: TestLocation(Uri.parse('file:///foo/setUpAll'), 345, 456), () {});
+        test('test 2 custom', location: TestLocation(Uri.parse('file:///foo/test'), 567, 789), () {});
+        tearDownAll(location: TestLocation(Uri.parse('file:///foo/tearDownAll'), 890, 901), () {});
+      });
+    ''', [
+        [
+          suiteJson(0),
+          testStartJson(1, 'loading test.dart', groupIDs: []),
+          testDoneJson(1, hidden: true),
+        ],
+        [
+          groupJson(2, testCount: 2),
+          groupJson(3,
+              name: 'group 1 inferred',
+              parentID: 2,
+              line: 6,
+              column: 7,
+              testCount: 1),
+          testStartJson(4, 'group 1 inferred (setUpAll)',
+              groupIDs: [2, 3], line: 7, column: 9),
+          testDoneJson(4, hidden: true),
+          testStartJson(5, 'group 1 inferred test 1 inferred',
+              groupIDs: [2, 3], line: 8, column: 9),
+          testDoneJson(5),
+          testStartJson(6, 'group 1 inferred (tearDownAll)',
+              groupIDs: [2, 3], line: 9, column: 9),
+          testDoneJson(6, hidden: true),
+          groupJson(7,
+              name: 'group 2 custom',
+              parentID: 2,
+              url: 'file:///foo/group',
+              line: 123,
+              column: 234,
+              testCount: 1),
+          testStartJson(8, 'group 2 custom (setUpAll)',
+              url: 'file:///foo/setUpAll',
+              groupIDs: [2, 7],
+              line: 345,
+              column: 456),
+          testDoneJson(8, hidden: true),
+          testStartJson(9, 'group 2 custom test 2 custom',
+              url: 'file:///foo/test',
+              groupIDs: [2, 7],
+              line: 567,
+              column: 789),
+          testDoneJson(9),
+          testStartJson(10, 'group 2 custom (tearDownAll)',
+              url: 'file:///foo/tearDownAll',
+              groupIDs: [2, 7],
+              line: 890,
+              column: 901),
+          testDoneJson(10, hidden: true),
+        ]
+      ], doneJson());
+    });
   });
 
   test(

pkgs/test/test/runner/json_reporter_utils.dart

@@ -89,13 +89,16 @@ Map<String, Object> groupJson(int id,
     int? parentID,
     Object? skip,
     int? testCount,
+    String? url,
     int? line,
     int? column}) {
   if ((line == null) != (column == null)) {
     throw ArgumentError(
         'line and column must either both be null or both be passed');
   }
 
+  url ??=
+      line == null ? null : p.toUri(p.join(d.sandbox, 'test.dart')).toString();
   return {
     'type': 'group',
     'group': {
@@ -107,17 +110,15 @@ Map<String, Object> groupJson(int id,
       'testCount': testCount ?? 1,
       'line': line,
       'column': column,
-      'url': line == null
-          ? null
-          : p.toUri(p.join(d.sandbox, 'test.dart')).toString()
+      'url': url
     }
   };
 }
 
 /// Returns the event emitted by the JSON reporter indicating that a test has
 /// begun running.
 ///
-/// If [parentIDs] is passed, it's the IDs of groups containing this test. If
+/// If [groupIDs] is passed, it's the IDs of groups containing this test. If
 /// [skip] is `true`, the test is expected to be marked as skipped without a
 /// reason. If it's a [String], the test is expected to be marked as skipped
 /// with that reason.

pkgs/test/test/runner/line_and_col_test.dart

@@ -56,13 +56,49 @@ void main() {
       await test.shouldExit(0);
     });
 
+    test('additionally selects test with matching custom location', () async {
+      await d.dir('test').create();
+      var testFileUri = Uri.file(d.file('test/aaa_test.dart').io.path);
+      var notTestFileUri = Uri.file(d.file('test/bbb.dart').io.path);
+      var testFilePackageRelativeUri =
+          Uri.parse('org-dartlang-app:///test/aaa_test.dart');
+      await d.file('test/aaa_test.dart', '''
+        import 'package:test/test.dart';
+
+        void main() {
+          test("a", () {}); // Line 4 from stack trace (match)
+
+          // Custom line 4 (match)
+          test("b", location: TestLocation(Uri.parse('$testFileUri'), 4, 0), () {});
+
+          // Custom line 4 match but using org-dartlang-app (match)
+          test("c", location: TestLocation(Uri.parse('$testFilePackageRelativeUri'), 4, 0), () {});
+
+          // Custom different line, same file (no match)
+          test("d", location: TestLocation(Uri.parse('$testFileUri'), 5, 0), () => throw TestFailure("oh no"));
+
+          // Custom line 4 match but different file (no match)
+          test("e", location: TestLocation(Uri.parse('$notTestFileUri'), 4, 0), () => throw TestFailure("oh no"));
+        }
+      ''').create();
+
+      var test = await runTest(['test/aaa_test.dart?line=4']);
+
+      expect(
+        test.stdout,
+        emitsThrough(contains('+3: All tests passed!')),
+      );
+
+      await test.shouldExit(0);
+    });
+
     test('selects groups with a matching line', () async {
       await d.file('test.dart', '''
         import 'package:test/test.dart';
 
         void main() {
           group("a", () {
-            test("b", () {});
+            test("a", () {});
           });
           group("b", () {
             test("b", () => throw TestFailure("oh no"));
@@ -80,6 +116,53 @@ void main() {
       await test.shouldExit(0);
     });
 
+    test('additionally selects groups with a matching custom location',
+        () async {
+      await d.dir('test').create();
+      var testFileUri = Uri.file(d.file('test/aaa_test.dart').io.path);
+      var notTestFileUri = Uri.file(d.file('test/bbb.dart').io.path);
+      var testFilePackageRelativeUri =
+          Uri.parse('org-dartlang-app:///test/aaa_test.dart');
+      await d.file('test/aaa_test.dart', '''
+        import 'package:test/test.dart';
+
+        void main() {
+          group("a", () { // Line 4 from stack trace (match)
+            test("a", () {});
+          });
+
+          // Custom line 4 (match)
+          group("b", location: TestLocation(Uri.parse('$testFileUri'), 4, 0), () {
+            test("b", () {});
+          });
+
+          // Custom line 4 match but using org-dartlang-app (match)
+          group("c", location: TestLocation(Uri.parse('$testFilePackageRelativeUri'), 4, 0), () {
+            test("c", () {});
+          });
+
+          // Custom different line, same file (no match)
+          group("d", location: TestLocation(Uri.parse('$testFileUri'), 5, 0), () {
+            test("d", () => throw TestFailure("oh no"));
+          });
+
+          // Custom line 4 match but different file (no match)
+          group("e", location: TestLocation(Uri.parse('$notTestFileUri'), 4, 0), () {
+            test("e", () => throw TestFailure("oh no"));
+          });
+        }
+      ''').create();
+
+      var test = await runTest(['test/aaa_test.dart?line=4']);
+
+      expect(
+        test.stdout,
+        emitsThrough(contains('+3: All tests passed!')),
+      );
+
+      await test.shouldExit(0);
+    });
+
     test('No matching tests', () async {
       await d.file('test.dart', '''
         import 'package:test/test.dart';

pkgs/test_api/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.7.5-wip
+
+* `test()` and `group()` functions now take an optional `TestLocation` that will
+  be used as the location of the test in JSON reporters instead of being parsed
+  from the call stack.
+
 ## 0.7.4
 
 * Allow `analyzer: '>=6.0.0 <8.0.0'`

pkgs/test_api/lib/backend.dart

@@ -11,3 +11,4 @@ export 'src/backend/runtime.dart' show Runtime;
 export 'src/backend/stack_trace_formatter.dart' show StackTraceFormatter;
 export 'src/backend/stack_trace_mapper.dart' show StackTraceMapper;
 export 'src/backend/suite_platform.dart' show SuitePlatform;
+export 'src/backend/test_location.dart' show TestLocation;

pkgs/test_api/lib/scaffolding.dart

@@ -16,6 +16,7 @@ export 'src/backend/configuration/skip.dart' show Skip;
 export 'src/backend/configuration/tags.dart' show Tags;
 export 'src/backend/configuration/test_on.dart' show TestOn;
 export 'src/backend/configuration/timeout.dart' show Timeout;
+export 'src/backend/test_location.dart' show TestLocation;
 export 'src/scaffolding/spawn_hybrid.dart' show spawnHybridCode, spawnHybridUri;
 export 'src/scaffolding/test_structure.dart'
     show addTearDown, group, setUp, setUpAll, tearDown, tearDownAll, test;