Switch to using Gradle Configurations for sharing data between projects (#21)

Fixes #18

Author: mcumings

README.md

@@ -47,7 +47,7 @@ extensions.configure<MetricsForDevelocityExtension> {
         // value is configured for this property then the Develocity key will be searched
         // for in the standard locations for manual key provisioning, documented here:
         // https://docs.gradle.com/develocity/gradle-plugin/current/#manual_access_key_configuration
-        develocityAccessKey.set("https://custom-develocity-server.com")
+        develocityAccessKey.set("your_base64_encoded_access_key")
 
         // Optional: Configure the query filter to use when querying the Develocity server for
         // builds.  This filter is expressed using the Develocity's advanced search syntax:

docs/Design.md

@@ -35,10 +35,30 @@ Time window tasks (e.g., last 7 days)
     - Loads the hourly summary files
     - reduces the summaries together, persisting them to disk by ID
 
-## Data consumer/reporting tasks
+All data gathering tasks are performed only in the root project.
+
+## Consumable configurations
+
+In order to expose the root project's data to the consuming projects the root project defines
+consumable configurations which export the data produced by the data gathering tasks to the
+consuming projects.  These configurations use the same naming conventions as the data gathering
+tasks.
+
+To simplify the chore of wiring up the consuming projects, the
+[TaskProviderExtensions](../src/main/kotlin/com/ebay/plugins/metrics/develocity/TaskProviderExtensions.kt)
+file provides a set of extension functions that can be used to wire up an individual sumarizer's output of the data
+gathering task to a consuming project's `TaskProvider<out MetricSummarizerTask>` instance.
+
+For more advanced use cases, the consuming project can create a resolvable Gradle `Configuration` to refer to
+the root project's configuration by name. When this approach is used, the resolvable configuration should specify the
+[SUMMARIZER_ATTRIBUTE](../src/main/kotlin/com/ebay/plugins/metrics/develocity/MetricsForDevelocityConstants.kt)
+with a value of [SUMMARIZER_ALL], which will result in a directory of all summarizer outputs.  To select an
+individual summarizer's output, the consuming project can leverage the
+[SummarizerSelectTransform](../src/main/kotlin/com/ebay/plugins/metrics/develocity/SummarizerSelectTransform.kt)
+artifact transform to get at the desired individual output file.  
+
+## Data consumer / reporting tasks
 
 Data consumer / reporting tasks are configured to consume the aggregated data from the first
 stage of the pipeline and produce data or reports in their final form.  These tasks are
 to be implemented as needed to satisfy the requirements of the project.
-
-

src/main/kotlin/com/ebay/plugins/metrics/develocity/MetricsForDevelocityConstants.kt

@@ -0,0 +1,31 @@
+package com.ebay.plugins.metrics.develocity
+
+import org.gradle.api.attributes.Attribute
+
+/**
+ * Constants defined by the plugin which may be used by consuming plugins or build scripts.
+ */
+object MetricsForDevelocityConstants {
+    /**
+     * The name that the [MetricsForDevelocityExtension] is registered under.
+     */
+    const val EXTENSION_NAME = "metricsForDevelocity"
+
+    /**
+     * The gradle property name used to configure the query filter.
+     */
+    const val QUERY_FILTER_PROPERTY = "metricsForDevelocityQueryFilter"
+
+    /**
+     * The variant attribute used to identify what summarizer data is being exported or resolved.  The special
+     * value of [SUMMARIZER_ALL] is used will result in a directory contianing all summarizer data.  Consumers
+     * can start with this and apply a [SummarizerSelectTransform] to filter down to a single summarizer output.
+     */
+    val SUMMARIZER_ATTRIBUTE = Attribute.of("com.ebay.metrics-for-develocity.summarizer", String::class.java)
+
+    /**
+     * The attribute value used to identify the configuration used to export all summarizer data to
+     * consuming projects.
+     */
+    val SUMMARIZER_ALL = "_all_"
+}

src/main/kotlin/com/ebay/plugins/metrics/develocity/MetricsForDevelocityInternalExtension.kt

@@ -1,5 +1,11 @@
 package com.ebay.plugins.metrics.develocity
 
+import com.ebay.plugins.metrics.develocity.MetricsForDevelocityConstants.EXTENSION_NAME
+import com.ebay.plugins.metrics.develocity.MetricsForDevelocityConstants.QUERY_FILTER_PROPERTY
+import com.ebay.plugins.metrics.develocity.MetricsForDevelocityConstants.SUMMARIZER_ALL
+import com.ebay.plugins.metrics.develocity.MetricsForDevelocityConstants.SUMMARIZER_ATTRIBUTE
+import com.ebay.plugins.metrics.develocity.NameUtil.DATETIME_TASK_PATTERN
+import com.ebay.plugins.metrics.develocity.NameUtil.DURATION_TASK_PATTERN
 import com.ebay.plugins.metrics.develocity.projectcost.ProjectCostPlugin
 import com.ebay.plugins.metrics.develocity.service.DevelocityBuildService
 import com.ebay.plugins.metrics.develocity.userquery.UserQueryPlugin
@@ -20,7 +26,7 @@ import javax.inject.Inject
  * Plugin implementation which defines tasks and configurations artifacts which are used to
  * generate aggregate metric data based upon Develocity build scans.
  */
-@Suppress("unused") // False positive
+@Suppress("unused", "UnstableApiUsage") // False positive
 internal class MetricsForDevelocityPlugin @Inject constructor(
     private val providerFactory: ProviderFactory
 ) : Plugin<Any> {
@@ -70,6 +76,8 @@ internal class MetricsForDevelocityPlugin @Inject constructor(
     }
 
     private fun applyProject(project: Project) {
+        project.dependencies.attributesSchema.attribute(SUMMARIZER_ATTRIBUTE)
+
         if (project.parent == null) {
             applyRootProject(project)
         }
@@ -135,11 +143,68 @@ internal class MetricsForDevelocityPlugin @Inject constructor(
             ext,
         )
 
+        /*
+        * Creates a rule for generating consumable configurations for date time requests.
+        * These can be scoped to daily or hourly time specifications.
+        */
+        project.configurations.addRule(
+            "Pattern: ${NameUtil.dateTime("<YYYY>-<MM>-<DD>[T<HH>]")}  " +
+                    "Gathers Develocity metrics for the date (and optionally hour) specified."
+        ) { configurationName ->
+            val matcher = DATETIME_TASK_PATTERN.matcher(configurationName)
+            if (!matcher.matches()) return@addRule
+
+            val date = matcher.group(2)
+            val timeSpec: String = matcher.group(1)
+            val hour: String? = matcher.group(3)
+
+            project.configurations.register(configurationName) { config ->
+                with(config) {
+                    isCanBeConsumed = true
+                    isCanBeResolved = false
+                    isTransitive = false
+                    attributes.attribute(SUMMARIZER_ATTRIBUTE, SUMMARIZER_ALL)
+                }
+            }
+
+            val artifactTaskProvider = if (hour == null) {
+                pluginContext.registerDaily(date)
+            } else {
+                pluginContext.registerHourly(timeSpec)
+            }
+            project.artifacts.add(configurationName, artifactTaskProvider)
+        }
+
+        /*
+         * Creates a rule for generating consumable configurations for duration-based requests.
+         * These can be scoped to specific time duration specifications.
+         */
+        project.configurations.addRule(
+            "Pattern: ${NameUtil.duration("<Java Duration String>")}  " +
+                    "Aggregates Develocity metrics for the current date back in time for the " +
+                    "specified duration."
+        ) { configurationName ->
+            val matcher = DURATION_TASK_PATTERN.matcher(configurationName)
+            if (!matcher.matches()) return@addRule
+
+            val durationStr: String = matcher.group(1)
+            val consumableConfig = project.configurations.register(configurationName)
+            consumableConfig.configure { config ->
+                with(config) {
+                    isCanBeConsumed = true
+                    isCanBeResolved = false
+                    isTransitive = false
+                    attributes.attribute(SUMMARIZER_ATTRIBUTE, SUMMARIZER_ALL)
+                }
+            }
+            project.artifacts.add(configurationName, pluginContext.registerDurationAggregation(durationStr))
+        }
+
         /*
          * Rule to register a task for a specific date and optionally hour.
          */
         project.tasks.addRule(
-            "Pattern: $TASK_PREFIX-<YYYY>-<MM>-<DD>T[<HH>]  " +
+            "Pattern: ${NameUtil.dateTime("<YYYY>-<MM>-<DD>[T<HH>]")}  " +
                     "Gathers Develocity metrics for the date (and optionally hour) specified."
         ) { taskName ->
             val matcher = DATETIME_TASK_PATTERN.matcher(taskName)
@@ -162,7 +227,7 @@ internal class MetricsForDevelocityPlugin @Inject constructor(
          * Rule to register a task for a time window / duration, relative to the current day and hour.
          */
         project.tasks.addRule(
-            "Pattern: $TASK_PREFIX-last-<Java Duration String>  " +
+            "Pattern: ${NameUtil.duration("<Java Duration String>")}  " +
                     "Aggregates Develocity metrics for the current date back in time for the " +
                     "specified duration."
         ) { taskName ->
@@ -172,22 +237,14 @@ internal class MetricsForDevelocityPlugin @Inject constructor(
             val durationStr: String = matcher.group(1)
             pluginContext.registerDurationAggregation(durationStr)
         }
-
-        ext.extensions.create(
-            INTERNAL_EXTENSION_NAME,
-            MetricsForDevelocityInternalExtension::class.java,
-            { timeSpec: String -> pluginContext.registerHourly(timeSpec) },
-            { dateStr: String -> pluginContext.registerDaily(dateStr) },
-            { durationStr: String -> pluginContext.registerDurationAggregation(durationStr) },
-        )
     }
 
     /*
      * Function to register an hourly gather task.  This function must be re-entrant and allow for
      * the same task to be registered multiple times without error.
      */
     private fun PluginContext.registerHourly(timeSpec: String): TaskProvider<GatherHourlyTask> {
-        val taskName = "$TASK_PREFIX-$timeSpec"
+        val taskName = NameUtil.dateTime(timeSpec)
         return if (project.tasks.names.contains(taskName)) {
             // task already exists.  Return its TaskProvider.
             project.tasks.named(taskName, GatherHourlyTask::class.java)
@@ -227,7 +284,7 @@ internal class MetricsForDevelocityPlugin @Inject constructor(
      * the same task to be registered multiple times without error.
      */
     private fun PluginContext.registerDaily(dateString: String): TaskProvider<GatherAggregateTask> {
-        val taskName = "$TASK_PREFIX-$dateString"
+        val taskName = NameUtil.dateTime(dateString)
         return if (project.tasks.names.contains(taskName)) {
             // task already exists.  Return its TaskProvider.
             project.tasks.named(taskName, GatherAggregateTask::class.java)
@@ -281,7 +338,7 @@ internal class MetricsForDevelocityPlugin @Inject constructor(
             }
         }.toList()
 
-        val taskName = "$TASK_PREFIX-last-$durationStr"
+        val taskName = NameUtil.duration(durationStr)
         return if (project.tasks.names.contains(taskName)) {
             // task already exists.  Return its TaskProvider.
             project.tasks.named(taskName, GatherAggregateTask::class.java)
@@ -357,25 +414,6 @@ internal class MetricsForDevelocityPlugin @Inject constructor(
     }
 
     companion object {
-        const val EXTENSION_NAME = "metricsForDevelocity"
-        const val INTERNAL_EXTENSION_NAME = "metricsForDevelocityInternal"
-        const val TASK_PREFIX = "metricsForDevelocity"
-        const val TASK_GROUP = "develocity metrics"
-        const val QUERY_FILTER_PROPERTY = "metricsForDevelocityQueryFilter"
-
-        private val DATETIME_TASK_PATTERN = Regex(
-            // Examples:
-            //      metricsForDevelocity-2024-06-01
-            //      metricsForDevelocity-2024-06-01T05
-            "^\\Q$TASK_PREFIX-\\E((\\d{4}-\\d{2}-\\d{2})(?:T(\\d{2}))?)$"
-        ).toPattern()
-
-        private val DURATION_TASK_PATTERN = Regex(
-            // Examples:
-            //      metricsForDevelocity-last-P7D
-            //      metricsForDevelocity-last-PT8H
-            //      metricsForDevelocity-last-P2DT8H
-            "^\\Q$TASK_PREFIX-last-\\E(\\w+)$"
-        ).toPattern()
+        internal const val TASK_GROUP = "develocity metrics"
     }
 }

src/main/kotlin/com/ebay/plugins/metrics/develocity/MetricsForDevelocityPlugin.kt

@@ -0,0 +1,27 @@
+package com.ebay.plugins.metrics.develocity
+
+/**
+ * Utility functions for generating task names, to keep it all in one place.
+ */
+internal object NameUtil {
+    const val TASK_PREFIX = "metricsForDevelocity"
+
+    val DATETIME_TASK_PATTERN = Regex(
+        // Examples:
+        //      metricsForDevelocity-2024-06-01
+        //      metricsForDevelocity-2024-06-01T05
+        "^\\Q$TASK_PREFIX-\\E((\\d{4}-\\d{2}-\\d{2})(?:T(\\d{2}))?)$"
+    ).toPattern()
+
+    val DURATION_TASK_PATTERN = Regex(
+        // Examples:
+        //      metricsForDevelocity-last-P7D
+        //      metricsForDevelocity-last-PT8H
+        //      metricsForDevelocity-last-P2DT8H
+        "^\\Q$TASK_PREFIX-last-\\E(\\w+)$"
+    ).toPattern()
+
+    fun dateTime(timeSpec: String): String = "$TASK_PREFIX-$timeSpec"
+
+    fun duration(durationStr: String): String = "$TASK_PREFIX-last-$durationStr"
+}

src/main/kotlin/com/ebay/plugins/metrics/develocity/NameUtil.kt

@@ -9,7 +9,7 @@ import org.gradle.api.provider.Provider
 /**
  * Utility to help consolidate all path-related logic for the develocity metrics plugin.
  */
-object PathUtil {
+internal object PathUtil {
     private fun summarizerFileName(summarizer: MetricSummarizer<*>) = summarizer.id
 
     /**
@@ -26,8 +26,8 @@ object PathUtil {
         projectLayout: ProjectLayout,
         dateStr: String,
     ): Provider<Directory> = dailyBaseDir(projectLayout).map { dailyBaseDir ->
-            dailyBaseDir.dir(dateStr)
-        }
+        dailyBaseDir.dir(dateStr)
+    }
 
     /**
      * The base directory for all hourly outputs.

src/main/kotlin/com/ebay/plugins/metrics/develocity/PathUtil.kt

@@ -0,0 +1,29 @@
+package com.ebay.plugins.metrics.develocity
+
+import org.gradle.api.artifacts.transform.InputArtifact
+import org.gradle.api.artifacts.transform.TransformAction
+import org.gradle.api.artifacts.transform.TransformOutputs
+import org.gradle.api.file.FileSystemLocation
+import org.gradle.api.provider.Provider
+import org.gradle.api.tasks.PathSensitive
+import org.gradle.api.tasks.PathSensitivity
+import org.gradle.work.DisableCachingByDefault
+
+/**
+ * Artifact transform which extracts an individual summarizer's result from the output of the summary
+ * computation task.
+ */
+@DisableCachingByDefault(because = "Copying files does not benefit from caching")
+abstract class SummarizerSelectTransform : TransformAction<SummarizerSelectTransformParameters> {
+    @get:InputArtifact
+    @get:PathSensitive(PathSensitivity.NONE)
+    abstract val inputArtifact: Provider<FileSystemLocation>
+
+    override fun transform(outputs: TransformOutputs) {
+        val input = inputArtifact.get().asFile
+        require(input.isDirectory) {
+            "Input artifact must be a directory, but was: $input"
+        }
+        outputs.file(input.resolve(parameters.summarizerId.get()))
+    }
+}

src/main/kotlin/com/ebay/plugins/metrics/develocity/SummarizerSelectTransform.kt

@@ -0,0 +1,16 @@
+package com.ebay.plugins.metrics.develocity
+
+import org.gradle.api.artifacts.transform.TransformParameters
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.Input
+
+/**
+ * Parameters for the summarizer selection transform.
+ */
+interface SummarizerSelectTransformParameters : TransformParameters {
+    /**
+     * The [MetricSummarizer.id] of the summarizer data to extract.
+     */
+    @get:Input
+    val summarizerId: Property<String>
+}