KTOR-7489 Multi-platform support + kotlinx-rpc (#158)

* Dependency resolution and compilation of all plugins are now handled through gradle during the build phase. This allows for highlighting and auto-complete to work in all the plugin source directories, using only the yaml manifests.
* Each plugin can have multiple modules defined, based on some pre-defined modules: server (default), core, and client. These are resolved as separate gradle modules with independent dependencies and sources.
* Modules can be referenced from the plugins for dependencies, gradle plugins, and source files. This controls where the generator will send these items.
* Deleted some 1.* files and removed support for the old JSON formats.
* Moved around tests and related artifacts because of more logic happening during the build phase.
* Added ability to disable build systems for plugins when unsupported (i.e., maven, amper)
* Moved version variables from yaml to plugins/gradle.properties, so it's easier for dependabot etc. Also added version references from manifest for gradle plugins
* Added kotlinx-rpc plugin

Author: bjhham

README.md

@@ -33,15 +33,28 @@ To add a new plugin, follow these easy steps:
    - You can include any number of install files for populating new projects.  More information under 
      [templates/manifest.ktor.yaml](templates/manifest.ktor.yaml).  The existing plugin files under the 
      [plugins](plugins) folder can also be useful reference for introducing new plugins.
-   - If your plugin artifacts aren't published on Maven Central you may need to add an entry to the `allRepositories()`
-     function in the [Repositories.kt](buildSrc/src/main/kotlin/io/ktor/plugins/registry/Repositories.kt) file.
    <br /><br />
    
 4. **Run `./gradlew buildRegistry`** to test the new files.<br /><br />
 
 5. **Run `./gradlew buildTestProject`** to generate a sample project.
    - You can now experiment with a project generated with your plugin.
-   - Iterate on updating the new files, building the registry, and generating the test project until you're happy with the results.
+   - Iterate on updating the new files, building the registry, and generating the test project until you're happy with the results.<br /><br />
 
 6. **Create a pull request** with the new changes.
-    - Once merged, your plugin will be available in the ktor project generator.
+    - Once merged, your plugin will be available in the ktor project generator.
+
+
+### Examples
+
+To supplement the [reference templates](templates), we have many plugin examples that can be of some aid when 
+adding your own.
+
+Here are a few examples to illustrate different techniques that you can apply for your extensions:
+
+| plugin                                                                      | features                                             |
+|-----------------------------------------------------------------------------|------------------------------------------------------|
+| [koin](plugins/server/io.insert-koin)                                       | Basic layout with an extra source file               |
+| [exposed](plugins/server/org.jetbrains/exposed)                             | Several injections and dependencies                  |
+| [ktor-server-kafka](plugins/server/io.github.flaxoos/ktor-server-kafka-jvm) | Includes configuration files and a custom repository |
+| [kotlinx-rpc](plugins/server/org.jetbrains/kotlinx-rpc)                     | Multi-module layout with core, client, and server    |                                              

build.gradle.kts

@@ -5,9 +5,11 @@
 @file:Suppress("UnstableApiUsage")
 
 import io.ktor.plugins.registry.*
-import java.nio.file.Paths
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
-val targets by lazy { fetchKtorTargets(logger) }
+val ktorReleases = getKtorReleases(logger)
+val latestKtor = ktorReleases.last()
+val pluginConfigs by lazy { collectPluginConfigs(logger, ktorReleases) }
 
 plugins {
     alias(libs.plugins.serialization)
@@ -18,30 +20,64 @@ plugins {
 group = "io.ktor"
 version = "1.0-SNAPSHOT"
 
+// create build config for each valid release-plugin-target triple
 configurations {
-    for (target in targets)
-        target.releaseConfigs.forEach(::create)
+    // create plugin build configs
+    for (pluginConfig in pluginConfigs) {
+        create(pluginConfig.name) {
+            pluginConfig.parent?.let { parent ->
+                extendsFrom(get(parent))
+            }
+        }
+    }
 }
 
-dependencies {
-    // each ktor version has its own classpath
-    for (target in targets) {
-        for ((config, version) in target.releases) {
-            config("io.ktor:ktor-${target.name}-core:$version")
-            config(kotlin("stdlib"))
-
-            // test imports for test_function template
-            if (target.name == "server") {
-                config(kotlin("test"))
-                config(kotlin("test-junit"))
-                config("io.ktor:ktor-server-test-host:$version")
+repositories {
+    mavenCentral()
+
+    for (repositoryUrl in pluginConfigs.flatMap { it.repositories }.distinct()) {
+        maven(repositoryUrl)
+    }
+}
+
+sourceSets {
+    // include all the plugins as source paths, using the latest valid ktor release for each
+    for (pluginConfig in pluginConfigs.latestByPath()) {
+        create(pluginConfig.name) {
+            kotlin.srcDir("plugins/${pluginConfig.path}")
+            compileClasspath += configurations[pluginConfig.name]
+            pluginConfig.parent?.let { parent ->
+                compileClasspath += configurations[parent]
+                compileClasspath += sourceSets[parent].output
             }
+        }
+    }
+}
 
-            for (dependency in target.allArtifactsForVersion(version))
-                config(dependency)
+dependencies {
+    // create a build config for every plugin-release-module combination
+    for (pluginConfig in pluginConfigs) {
+        val type = pluginConfig.type
+        val release = pluginConfig.release
+        val config = pluginConfig.name
+
+        // common dependencies
+        config(kotlin("stdlib"))
+        config(kotlin("test"))
+        config(kotlin("test-junit"))
+        config("io.ktor:ktor-$type-core:$release")
+        when(type) {
+            "client" -> config("io.ktor:ktor-client-mock:$release")
+            "server" -> config("io.ktor:ktor-server-test-host:$release")
         }
+
+        // artifacts for the specific plugin version
+        for ((group, name, version) in pluginConfig.artifacts)
+            config("$group:$name:${version.resolvedString}")
     }
-    val latestKtor = targets.first().releases.last().version
+
+    // shared sources used in buildSrc
+    implementation(files("buildSrc/build/libs/shared.jar"))
 
     // current ktor dependencies for handling manifests
     implementation("io.ktor:ktor-server-core:$latestKtor")
@@ -67,15 +103,6 @@ dependencies {
     testImplementation(kotlin("test"))
 }
 
-// include relevant copied classes from buildSrc module
-sourceSets {
-    main {
-        kotlin {
-            srcDir("build/copied")
-        }
-    }
-}
-
 detekt {
     toolVersion = libs.versions.detekt.version.get()
     config.setFrom(file("detekt.yml"))
@@ -88,19 +115,6 @@ tasks {
         useJUnitPlatform()
     }
 
-    /**
-     *  We copy this shared source file with the parent project because there is otherwise
-     *  a chicken/egg problem with building the project which confuses the IDEA.
-     */
-    val copyPluginTypes by registering(Copy::class) {
-        group = "build"
-        from(
-            "buildSrc/src/main/kotlin/io/ktor/plugins/registry/PluginReference.kt",
-            "buildSrc/src/main/kotlin/io/ktor/plugins/registry/PluginCollector.kt",
-        )
-        into("build/copied")
-    }
-
     // download all sources
     compileKotlin {
         compilerOptions {
@@ -109,40 +123,15 @@ tasks {
                 "-XdownloadSources=true"
             )
         }
-        dependsOn(copyPluginTypes)
     }
 
     // resolving plugin jars from custom classpaths
     val resolvePlugins by registering {
         group = "plugins"
         description = "Locate plugin resources from version definitions"
         doLast {
-            for (target in targets) {
-                val resolvedArtifacts = target.releases.associate { release ->
-                    release.version to configurations[release.config].resolvedConfiguration.resolvedArtifacts
-                }
-                outputReleaseArtifacts(
-                    outputFile = Paths.get("build/${target.name}-artifacts.yaml"),
-                    configurations = resolvedArtifacts
-                )
-            }
-        }
-    }
-
-    // print jar origins for finding problematic classpath imports
-    val outputDependencies by registering {
-        group = "plugins"
-        description = "Print all artifacts and their origins to deps-server.txt and deps-client.txt"
-        dependsOn(resolvePlugins)
-        doLast {
-            val reportDir = Paths.get("${project.rootDir.absolutePath}/dependencies")
-            prepareDirectory(reportDir)
-
-            for (target in targets) {
-                val resolvedArtifacts = target.releases.associate { release ->
-                    release.version to configurations[release.config].resolvedConfiguration
-                }
-                outputDependencyTrees(reportDir.resolve(target.name), resolvedArtifacts)
+            writeResolvedPluginConfigurations(pluginConfigs) { configName ->
+                configurations[configName].resolvedConfiguration
             }
         }
     }
@@ -156,13 +145,20 @@ tasks {
         standardInput = System.`in`
     }
 
+    // compiles ALL build targets before building the registry
+    val compileAll by registering {
+        group = "build"
+        description = "Compile all source sets"
+        dependsOn(withType<KotlinCompile>())
+    }
+
     // builds the registry for distributing to the project generator
     val buildRegistry by registering(JavaExec::class) {
         group = "plugins"
         description = "Build the registry from plugin resources"
         mainClass = "io.ktor.plugins.registry.BuildRegistryKt"
         classpath = sourceSets["main"].runtimeClasspath
-        dependsOn(resolvePlugins)
+        dependsOn(resolvePlugins, compileAll)
     }
 
     // generates a test project using the modified plugins in the repository

buildSrc/build.gradle.kts

@@ -1,18 +1,67 @@
 /*
  * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
+@file:Suppress("UnstableApiUsage")
 
 plugins {
     `kotlin-dsl`
     alias(libs.plugins.serialization)
 }
 
+val shared = "shared"
+
+configurations {
+    create(shared)
+}
+
 dependencies {
+    shared(kotlin("stdlib"))
+    shared(libs.kotlinx.serialization.core)
+    shared(libs.maven.artifact)
+
     implementation(libs.kaml)
     implementation(libs.maven.artifact)
+
     testImplementation(kotlin("test"))
 }
 
+sourceSets {
+    val sharedSrc = create(shared) {
+        kotlin.srcDir("../shared")
+        compileClasspath += configurations[shared]
+    }
+    main {
+        compileClasspath += sharedSrc.output
+        runtimeClasspath += sharedSrc.output
+    }
+    test {
+        compileClasspath += sharedSrc.output
+        runtimeClasspath += sharedSrc.output
+    }
+}
+
+val packageShared = tasks.register<Jar>("packageShared") {
+    archiveBaseName.set("shared")
+    from(sourceSets["shared"].output)
+}
+
+// Task to copy output of shared source set to main output
+val copySharedOutput = tasks.register<Copy>("copySharedOutput") {
+    from(sourceSets["shared"].output)
+    into(sourceSets["main"].output.classesDirs.first { it.toString().contains("kotlin") })
+    finalizedBy(packageShared)
+}
+
+tasks.named("compileKotlin").configure {
+    finalizedBy(copySharedOutput)
+}
+
+for (task in listOf("jar", "pluginUnderTestMetadata", "validatePlugins").map { tasks.named(it) }) {
+    task.configure {
+        mustRunAfter(copySharedOutput)
+    }
+}
+
 tasks.withType<Test> {
     useJUnitPlatform()
 }

buildSrc/src/main/kotlin/io/ktor/plugins/registry/Files.kt

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package io.ktor.plugins.registry
+
+import com.charleskorn.kaml.Yaml
+import com.charleskorn.kaml.YamlMap
+import com.charleskorn.kaml.parseToYamlNode
+import com.charleskorn.kaml.yamlMap
+import java.nio.file.Path
+import java.nio.file.Paths
+import kotlin.io.path.ExperimentalPathApi
+import kotlin.io.path.createDirectory
+import kotlin.io.path.deleteRecursively
+import kotlin.io.path.exists
+import kotlin.io.path.inputStream
+import kotlin.io.path.isDirectory
+import kotlin.io.path.listDirectoryEntries
+
+// Get all directories using patterns like "plugins/client/*/*"
+fun folders(pattern: String): List<Path> {
+    val segments = pattern.split("/").iterator()
+    if (!segments.hasNext())
+        return emptyList()
+
+    var folders = listOf<Path>(Paths.get(segments.next().ifEmpty { "/" }))
+    while(segments.hasNext()) {
+        val next = segments.next()
+        folders = when(next) {
+            "*" -> folders.flatMap { folder ->
+                when {
+                    folder.isDirectory() -> folder.listDirectoryEntries()
+                    else -> emptyList()
+                }
+            }
+            else -> folders.mapNotNull { folder -> folder.resolve(Paths.get(next)).takeIf { it.exists() } }
+        }
+    }
+    return folders.filter { it.isDirectory() }
+}
+
+@OptIn(ExperimentalPathApi::class)
+fun Path.clear(): Path {
+    deleteRecursively()
+    return createDirectory()
+}
+
+fun Path.readYamlMap(): YamlMap? =
+    try {
+        takeIf { it.exists() }
+            ?.inputStream()
+            ?.use(Yaml.default::parseToYamlNode)
+            ?.yamlMap
+    } catch (_: Exception) {
+        null
+    }