Skip to content

Commit

Permalink
Fix build scripts to distribute the right files (#10297)
Browse files Browse the repository at this point in the history
Huge, but mostly boring PR fixing a bunch of just necessary things.
  • Loading branch information
snazy authored Jan 29, 2025
1 parent edd79d4 commit 060c0a3
Show file tree
Hide file tree
Showing 16 changed files with 17,260 additions and 2,327 deletions.
2,319 changes: 35 additions & 2,284 deletions LICENSE

Large diffs are not rendered by default.

16,636 changes: 16,636 additions & 0 deletions LICENSE-BINARY-DIST

Large diffs are not rendered by default.

20 changes: 14 additions & 6 deletions NOTICE
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
Dremio
Copyright 2015-2017 Dremio Corporation
Copyright 2015-2025 Dremio Corporation

This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
Expand All @@ -25,17 +25,25 @@ Copyright The Apache Software Foundation
Apache Kerby
Copyright The Apache Software Foundation

Apache Polaris (incubating)
Copyright The Apache Software Foundation

Netty
Copyright The Netty Project

This product includes additional software licensed under the terms
of the following licenses, see LICENSE file:
of the following licenses, see LICENSE and LICENSE-BINARY-DIST file:
* Apache Software License, Version 2.0
* Creative Commons 1.0 Universal
* Eclipse Distribution License, Version 1.0
* Eclipse Public License, Version 1.0
* Eclipse Public License, Version 2.0
* GNU General Public License, Version 2 with the GNU Classpath Exception

---
The full list of dependencies contained in a Nessie distributions is
mentioned with their concrete licenses in the LICENSE (source distribution)
and LICENSE-BINARY-DIST (binary distributions) files.

The main docs page for a particular Nessie release under
https://projectnessie.org/docs/ contains an aggregated
license report (since Nessie version 0.83).
The Nessie web site provides aggregated license reports for recent Nessie
releases, available at https://projectnessie.org/docs/ and
https://projectnessie.org/downloads/.
84 changes: 57 additions & 27 deletions build-logic/src/main/kotlin/LicenseFileValidation.kt
Original file line number Diff line number Diff line change
Expand Up @@ -21,28 +21,46 @@ import java.io.File
import org.gradle.api.GradleException

/**
* Validates that all dependencies with MIT/BSD/Go/UPL/ISC licenses, which do not have an Apache
* license, are mentioned in the `LICENSE` file.
* Validates that all dependencies with MIT/BSD/Go/UPL/ISC licenses, and Apache license, are
* mentioned in the `LICENSE` file.
*/
class LicenseFileValidation : DependencyFilter {
fun needsNoMention(license: String?): Boolean = license != null && (license.contains("Apache"))
val needsApacheLicenseMention = setOf("Apache")

fun needsMention(license: String?): Boolean =
license != null &&
(license.contains("MIT") ||
license.contains("BSD") ||
license.contains("Go") ||
license.contains("ISC") ||
license.contains("Universal Permissive"))
val needsFullLicenseMention = setOf("MIT", "BSD", "Go", "ISC", "Universal Permissive")

fun doesNeedApacheMention(licenses: List<String?>): Boolean {
for (license in licenses) {
if (license != null) {
if (needsApacheLicenseMention.any { license.contains(it) }) {
return true
}
}
}
return false
}

fun doesNeedFullMention(licenses: List<String?>): Boolean {
for (license in licenses) {
if (license != null) {
if (needsFullLicenseMention.any { license.contains(it) }) {
return true
}
}
}
// no licenses !
return true
}

override fun filter(data: ProjectData?): ProjectData {
data!!

val rootNoticeFile = data.project.rootProject.file("LICENSE").readText()
val rootLicenseFile = data.project.rootProject.file("LICENSE-BINARY-DIST").readText()

val licenseReport = data.project.extensions.getByType(LicenseReportExtension::class.java)

val missing = mutableMapOf<String, String>()
val missingApacheMentions = mutableSetOf<String>()
val missingFullMentions = mutableMapOf<String, String>()

data.allDependencies.forEach { mod ->
val licenses =
Expand All @@ -51,31 +69,43 @@ class LicenseFileValidation : DependencyFilter {
mod.poms.flatMap { it.licenses }.map { it.name })
.distinct()

if (!licenses.any { needsNoMention(it) } && licenses.any { needsMention(it) }) {
val groupModule = "${mod.group}:${mod.name}"
if (!rootNoticeFile.contains(groupModule)) {
missing.put(
"${mod.group}:${mod.name}",
val groupModule = "${mod.group}:${mod.name}"
val groupModuleRegex = "^$groupModule$".toRegex(RegexOption.MULTILINE)
if (!groupModuleRegex.containsMatchIn(rootLicenseFile)) {
if (doesNeedApacheMention(licenses)) {
missingApacheMentions.add(groupModule)
} else if (doesNeedFullMention(licenses)) {
missingFullMentions[groupModule] =
"""
---
${mod.group}:${mod.name}
$groupModule
${mod.licenseFiles.flatMap { it.fileDetails }.filter { it.file != null }.map { it.file }
.map { File("${licenseReport.absoluteOutputDir}/$it").readText().trim() }
.distinct()
.map { "\n\n$it\n" }
.joinToString("\n")
${
mod.licenseFiles.flatMap { it.fileDetails }.filter { it.file != null }.map { it.file }
.map { File("${licenseReport.absoluteOutputDir}/$it").readText().trim() }
.distinct().joinToString("\n") { "\n\n$it\n" }
}
"""
.trimIndent(),
)
.trimIndent()
}
}
}

if (!missing.isEmpty()) {
val missingError = StringBuilder()
if (!missingApacheMentions.isEmpty()) {
missingError.append("\n\nMissing Apache License mentions:")
missingError.append("\n--------------------------------\n")
missingApacheMentions.sorted().forEach { missingError.append("\n$it") }
}
if (!missingFullMentions.isEmpty()) {
missingError.append("\n\nMissing full license mentions:")
missingError.append("\n------------------------------\n")
missingFullMentions.toSortedMap().values.forEach { missingError.append("\n$it") }
}
if (!missingApacheMentions.isEmpty() || !missingFullMentions.isEmpty()) {

throw GradleException(
"License information for the following artifacts is missing in the root LICENSE file: ${missing.map { it.value }.joinToString("\n")}"
"License information for the following artifacts is missing in the root LICENSE-BINARY-DIST file: $missingError"
)
}

Expand Down
104 changes: 104 additions & 0 deletions build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerExtension.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/*
* Copyright (C) 2024 Dremio
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package copiedcode

// Marker for Nessie LICENSE file - keep it
// CODE_COPIED_TO_NESSIE

import org.gradle.api.Project
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.file.SourceDirectorySet
import org.gradle.api.provider.Property
import org.gradle.api.provider.SetProperty

abstract class CopiedCodeCheckerExtension(private val project: Project) {
/**
* Per-project set of additional directories to scan.
*
* This property is _not_ propagated to subprojects.
*/
val scanDirectories =
project.objects.domainObjectContainer(
SourceDirectorySet::class.java,
{ name -> project.objects.sourceDirectorySet(name, name) },
)

/**
* By default, this plugin scans all files. The content types that match the regular expression of
* this property are excluded, unless a content-type matches one of the regular expressions in
* [CopiedCodeCheckerExtension.includedContentTypePatterns].
*
* See [CopiedCodeCheckerExtension.addDefaultContentTypes],
* [CopiedCodeCheckerExtension.includedContentTypePatterns],
* [CopiedCodeCheckerExtension.includeUnrecognizedContentType],
* [CopiedCodeCheckerExtension.includedContentTypePatterns].
*/
abstract val excludedContentTypePatterns: SetProperty<String>
/**
* By default, this plugin scans all files. The content types that match the regular expression of
* the [copiedcode.CopiedCodeCheckerExtension.excludedContentTypePatterns] property are excluded,
* unless a content-type matches one of the regular expressions in this property.
*
* See [CopiedCodeCheckerExtension.addDefaultContentTypes],
* [CopiedCodeCheckerExtension.excludedContentTypePatterns],
* [CopiedCodeCheckerExtension.includeUnrecognizedContentType],
* [CopiedCodeCheckerExtension.includedContentTypePatterns].
*/
abstract val includedContentTypePatterns: SetProperty<String>

/**
* If a content-type could not be detected, this property, which defaults to `true`, is consulted.
*
* See [CopiedCodeCheckerPlugin] for details.
*/
abstract val includeUnrecognizedContentType: Property<Boolean>

/**
* The magic "word", if present in a file, meaning "this file has been copied".
*
* A file is considered as "copied" must contain this magic word. "Word" means that the value must
* be surrounded by regular expression word boundaries (`\b`).
*/
abstract val magicWord: Property<String>

/**
* License file to check, configured on the root project. See [CopiedCodeCheckerPlugin] for
* details.
*/
abstract val licenseFile: RegularFileProperty

/** Recommended to use, adds known and used binary content types. */
fun addDefaultContentTypes(): CopiedCodeCheckerExtension {
// Exclude all images
excludedContentTypePatterns.add("image/.*")
// But include images built in XML (e.g. image/svg+xml)
includedContentTypePatterns.add("\\+xml")

return this
}

init {
includeUnrecognizedContentType.convention(true)
magicWord.convention(DEFAULT_MAGIC_WORD)
}

companion object {
// String manipulation is intentional - otherwise this source file would be considered as
// "copied".
val DEFAULT_MAGIC_WORD = "_CODE_COPIED_TO_NESSIE".substring(1)
}
}
Loading

0 comments on commit 060c0a3

Please sign in to comment.