From 9ff20be6edff182b1bae9d11a6e312a343005fd2 Mon Sep 17 00:00:00 2001 From: webwarrior Date: Tue, 5 Nov 2024 12:36:48 +0100 Subject: [PATCH 1/2] FileConventions.Test: test for versions in env Added a failing test for finding inconsistent versions in Github CI workflows when versions are stored in env vars. --- .../DummyCIWithPulumiVersionInEnvV2.0.0.yml | 17 +++++++++++++++++ .../DummyCIWithPulumiVersionInEnvV2.0.1.yml | 17 +++++++++++++++++ .../FileConventions.Test.fs | 8 ++++++++ 3 files changed, 42 insertions(+) create mode 100644 src/FileConventions.Test/DummyFiles/DummyWorkflowsWithEnv/DummyCIWithPulumiVersionInEnvV2.0.0.yml create mode 100644 src/FileConventions.Test/DummyFiles/DummyWorkflowsWithEnv/DummyCIWithPulumiVersionInEnvV2.0.1.yml diff --git a/src/FileConventions.Test/DummyFiles/DummyWorkflowsWithEnv/DummyCIWithPulumiVersionInEnvV2.0.0.yml b/src/FileConventions.Test/DummyFiles/DummyWorkflowsWithEnv/DummyCIWithPulumiVersionInEnvV2.0.0.yml new file mode 100644 index 00000000..5c64a87b --- /dev/null +++ b/src/FileConventions.Test/DummyFiles/DummyWorkflowsWithEnv/DummyCIWithPulumiVersionInEnvV2.0.0.yml @@ -0,0 +1,17 @@ +name: CI + +on: [push, pull_request] + +env: + PULUMI_VERSION: 2.0.0 + +jobs: + jobA: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - name: Setup Pulumi CLI + with: + pulumi-version: ${{ PULUMI_VERSION }} + - name: Print "Hello World!" + run: echo "Hello World!" diff --git a/src/FileConventions.Test/DummyFiles/DummyWorkflowsWithEnv/DummyCIWithPulumiVersionInEnvV2.0.1.yml b/src/FileConventions.Test/DummyFiles/DummyWorkflowsWithEnv/DummyCIWithPulumiVersionInEnvV2.0.1.yml new file mode 100644 index 00000000..b9fae26a --- /dev/null +++ b/src/FileConventions.Test/DummyFiles/DummyWorkflowsWithEnv/DummyCIWithPulumiVersionInEnvV2.0.1.yml @@ -0,0 +1,17 @@ +name: CI + +on: [push, pull_request] + +env: + PULUMI_VERSION: 2.0.1 + +jobs: + jobA: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - name: Setup Pulumi CLI + with: + pulumi-version: ${{ PULUMI_VERSION }} + - name: Print "Hello World!" + run: echo "Hello World!" diff --git a/src/FileConventions.Test/FileConventions.Test.fs b/src/FileConventions.Test/FileConventions.Test.fs index e32fe7a4..d36b2165 100644 --- a/src/FileConventions.Test/FileConventions.Test.fs +++ b/src/FileConventions.Test/FileConventions.Test.fs @@ -420,6 +420,14 @@ let DetectInconsistentVersionsInGitHubCI1() = Assert.That(DetectInconsistentVersionsInGitHubCI fileInfo, Is.EqualTo true) +[] +let DetectInconsistentVersionsInGitHubCI2() = + let fileInfo = + DirectoryInfo( + Path.Combine(dummyFilesDirectory.FullName, "DummyWorkflowsWithEnv") + ) + + Assert.That(DetectInconsistentVersionsInGitHubCI fileInfo, Is.EqualTo true) [] let DetectInconsistentVersionsInNugetRefsInFSharpScripts1() = From 3fd9b4988acb75260580f9f53ad716e52a938598 Mon Sep 17 00:00:00 2001 From: webwarrior Date: Tue, 5 Nov 2024 14:40:45 +0100 Subject: [PATCH 2/2] FileConventions: impl checking versions in env Implment finding inconsistent versions in Github CI workflows when versions are stored in env vars. This needs parsing of yaml files, for that YamlDotNet library was used. Also added reference to YamlDotNet in scripts that depend on this library. --- scripts/eofConvention.fsx | 1 + scripts/executableConvention.fsx | 1 + .../inconsistentVersionsInFSharpScripts.fsx | 1 + scripts/inconsistentVersionsInGitHubCI.fsx | 1 + scripts/mixedLineEndings.fsx | 1 + .../nonVerboseFlagsInGitHubCIAndScripts.fsx | 1 + scripts/shebangConvention.fsx | 1 + scripts/unpinnedDotnetToolInstallVersions.fsx | 1 + .../unpinnedGitHubActionsImageVersions.fsx | 1 + ...ackageReferenceVersionsInFSharpScripts.fsx | 1 + ...ugetPackageReferenceVersionsInProjects.fsx | 1 + scripts/wrapLatestCommitMsg.fsx | 1 + src/FileConventions/FileConventions.fsproj | 1 + src/FileConventions/Library.fs | 105 ++++++++++++++++-- 14 files changed, 108 insertions(+), 10 deletions(-) diff --git a/scripts/eofConvention.fsx b/scripts/eofConvention.fsx index 3f2b712b..fee53618 100755 --- a/scripts/eofConvention.fsx +++ b/scripts/eofConvention.fsx @@ -4,6 +4,7 @@ open System.IO open System #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1" +#r "nuget: YamlDotNet, Version=16.1.3" #load "../src/FileConventions/Helpers.fs" #load "../src/FileConventions/Library.fs" diff --git a/scripts/executableConvention.fsx b/scripts/executableConvention.fsx index cecaf6b5..b8d253cc 100755 --- a/scripts/executableConvention.fsx +++ b/scripts/executableConvention.fsx @@ -4,6 +4,7 @@ open System open System.IO #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1" +#r "nuget: YamlDotNet, Version=16.1.3" #load "../src/FileConventions/Library.fs" #load "../src/FileConventions/Helpers.fs" diff --git a/scripts/inconsistentVersionsInFSharpScripts.fsx b/scripts/inconsistentVersionsInFSharpScripts.fsx index 93922496..5458c12e 100755 --- a/scripts/inconsistentVersionsInFSharpScripts.fsx +++ b/scripts/inconsistentVersionsInFSharpScripts.fsx @@ -4,6 +4,7 @@ open System.IO open System.Linq #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1" +#r "nuget: YamlDotNet, Version=16.1.3" #load "../src/FileConventions/Library.fs" #load "../src/FileConventions/Helpers.fs" diff --git a/scripts/inconsistentVersionsInGitHubCI.fsx b/scripts/inconsistentVersionsInGitHubCI.fsx index f8041a45..84098ba9 100755 --- a/scripts/inconsistentVersionsInGitHubCI.fsx +++ b/scripts/inconsistentVersionsInGitHubCI.fsx @@ -3,6 +3,7 @@ open System.IO #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1" +#r "nuget: YamlDotNet, Version=16.1.3" #load "../src/FileConventions/Library.fs" #load "../src/FileConventions/Helpers.fs" diff --git a/scripts/mixedLineEndings.fsx b/scripts/mixedLineEndings.fsx index 427826a6..208b0dca 100755 --- a/scripts/mixedLineEndings.fsx +++ b/scripts/mixedLineEndings.fsx @@ -4,6 +4,7 @@ open System open System.IO #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1" +#r "nuget: YamlDotNet, Version=16.1.3" #load "../src/FileConventions/Library.fs" #load "../src/FileConventions/Helpers.fs" diff --git a/scripts/nonVerboseFlagsInGitHubCIAndScripts.fsx b/scripts/nonVerboseFlagsInGitHubCIAndScripts.fsx index 54bd123d..f89eb79d 100755 --- a/scripts/nonVerboseFlagsInGitHubCIAndScripts.fsx +++ b/scripts/nonVerboseFlagsInGitHubCIAndScripts.fsx @@ -4,6 +4,7 @@ open System open System.IO #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1" +#r "nuget: YamlDotNet, Version=16.1.3" #load "../src/FileConventions/Library.fs" #load "../src/FileConventions/Helpers.fs" diff --git a/scripts/shebangConvention.fsx b/scripts/shebangConvention.fsx index 965a9aad..ceb39ddf 100755 --- a/scripts/shebangConvention.fsx +++ b/scripts/shebangConvention.fsx @@ -4,6 +4,7 @@ open System open System.IO #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1" +#r "nuget: YamlDotNet, Version=16.1.3" #load "../src/FileConventions/Library.fs" #load "../src/FileConventions/Helpers.fs" diff --git a/scripts/unpinnedDotnetToolInstallVersions.fsx b/scripts/unpinnedDotnetToolInstallVersions.fsx index 92f35dcc..304a66db 100755 --- a/scripts/unpinnedDotnetToolInstallVersions.fsx +++ b/scripts/unpinnedDotnetToolInstallVersions.fsx @@ -4,6 +4,7 @@ open System open System.IO #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1" +#r "nuget: YamlDotNet, Version=16.1.3" #load "../src/FileConventions/Library.fs" #load "../src/FileConventions/Helpers.fs" diff --git a/scripts/unpinnedGitHubActionsImageVersions.fsx b/scripts/unpinnedGitHubActionsImageVersions.fsx index 07d31cc9..1b30c0a5 100755 --- a/scripts/unpinnedGitHubActionsImageVersions.fsx +++ b/scripts/unpinnedGitHubActionsImageVersions.fsx @@ -4,6 +4,7 @@ open System open System.IO #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1" +#r "nuget: YamlDotNet, Version=16.1.3" #load "../src/FileConventions/Library.fs" #load "../src/FileConventions/Helpers.fs" diff --git a/scripts/unpinnedNugetPackageReferenceVersionsInFSharpScripts.fsx b/scripts/unpinnedNugetPackageReferenceVersionsInFSharpScripts.fsx index 72e0e74b..d48fc1e2 100755 --- a/scripts/unpinnedNugetPackageReferenceVersionsInFSharpScripts.fsx +++ b/scripts/unpinnedNugetPackageReferenceVersionsInFSharpScripts.fsx @@ -4,6 +4,7 @@ open System open System.IO #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1" +#r "nuget: YamlDotNet, Version=16.1.3" #load "../src/FileConventions/Library.fs" #load "../src/FileConventions/Helpers.fs" diff --git a/scripts/unpinnedNugetPackageReferenceVersionsInProjects.fsx b/scripts/unpinnedNugetPackageReferenceVersionsInProjects.fsx index 56da33c0..b2e9fcd4 100755 --- a/scripts/unpinnedNugetPackageReferenceVersionsInProjects.fsx +++ b/scripts/unpinnedNugetPackageReferenceVersionsInProjects.fsx @@ -4,6 +4,7 @@ open System open System.IO #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1" +#r "nuget: YamlDotNet, Version=16.1.3" #load "../src/FileConventions/Library.fs" #load "../src/FileConventions/Helpers.fs" diff --git a/scripts/wrapLatestCommitMsg.fsx b/scripts/wrapLatestCommitMsg.fsx index 1672f418..8e7f6504 100755 --- a/scripts/wrapLatestCommitMsg.fsx +++ b/scripts/wrapLatestCommitMsg.fsx @@ -6,6 +6,7 @@ open System.Text.RegularExpressions open System.Linq #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1" +#r "nuget: YamlDotNet, Version=16.1.3" #load "../src/FileConventions/Library.fs" diff --git a/src/FileConventions/FileConventions.fsproj b/src/FileConventions/FileConventions.fsproj index 9c8c6d17..31b38ef5 100644 --- a/src/FileConventions/FileConventions.fsproj +++ b/src/FileConventions/FileConventions.fsproj @@ -13,6 +13,7 @@ + diff --git a/src/FileConventions/Library.fs b/src/FileConventions/Library.fs index e3017ed2..4cf74aaa 100644 --- a/src/FileConventions/Library.fs +++ b/src/FileConventions/Library.fs @@ -7,6 +7,7 @@ open System.Text.RegularExpressions open Mono open Mono.Unix.Native +open YamlDotNet.RepresentationModel let SplitByEOLs (text: string) (numberOfEOLs: uint) = if numberOfEOLs = 0u then @@ -300,21 +301,105 @@ let private DetectInconsistentVersions |> Seq.map(fun item -> Seq.length item.Value > 1) |> Seq.contains true +let private DetectInconsistentVersionsInYamlFiles + (fileInfos: seq) + (extractVersionsFunction: YamlNode -> seq) + = + let envVarRegex = + Regex(@"\s*\$\{\{\s*([^\s\}]+)\s*\}\}\s*", RegexOptions.Compiled) + + let yamlDocuments = + Seq.map + (fun (fileInfo: FileInfo) -> + let yaml = YamlStream() + use reader = new StreamReader(fileInfo.FullName) + yaml.Load reader + yaml.Documents[0].RootNode + ) + fileInfos + + let versionMap = + Seq.fold + (fun mapping (yamlDoc: YamlNode) -> + let matches = + Seq.collect extractVersionsFunction yamlDoc.AllNodes + + matches + |> Seq.fold + (fun acc (key, value) -> + let actualValue = + let variableRegexMatch = envVarRegex.Match value + + if variableRegexMatch.Success then + let yamlDict = yamlDoc :?> YamlMappingNode + + let envDict = + yamlDict.Children.["env"] + :?> YamlMappingNode + + let envVarName = + variableRegexMatch.Groups.[1].Value + + match envDict.Children.TryGetValue envVarName + with + | true, envVarValue -> + (envVarValue :?> YamlScalarNode).Value + | false, _ -> + failwithf "env. var %s not found" envVarName + else + value + + match Map.tryFind key acc with + | Some prevSet -> + Map.add key (Set.add actualValue prevSet) acc + | None -> Map.add key (Set.singleton actualValue) acc + ) + mapping + ) + Map.empty + yamlDocuments + + versionMap + |> Seq.map(fun item -> Seq.length item.Value > 1) + |> Seq.contains true + let DetectInconsistentVersionsInGitHubCIWorkflow(fileInfos: seq) = fileInfos |> Seq.iter(fun fileInfo -> assert (fileInfo.FullName.EndsWith ".yml")) - let inconsistentVersionsType1 = - DetectInconsistentVersions - fileInfos - "\\swith:\\s*([^\\s]*)-version:\\s*([^\\s]*)\\s" - - let inconsistentVersionsType2 = - DetectInconsistentVersions - fileInfos - "\\suses:\\s*([^\\s]*)@v([^\\s]*)\\s" + let extractVersions(node: YamlNode) = + match node with + | :? YamlMappingNode as yamlDict -> + yamlDict.Children + |> Seq.collect(fun (KeyValue(keyNode, valueNode)) -> + match keyNode, valueNode with + | (:? YamlScalarNode as keyScalar), + (:? YamlScalarNode as valueScalar) when + keyScalar.Value = "uses" + -> + match valueScalar.Value.Split "@v" with + | [| name; version |] -> Seq.singleton(name, version) + | _ -> Seq.empty + | (:? YamlScalarNode as keyScalar), + (:? YamlMappingNode as valueMapping) when + keyScalar.Value = "with" + -> + valueMapping.Children + |> Seq.choose(fun (KeyValue(innerKeyNode, innerValueNode)) -> + match innerKeyNode, innerValueNode with + | (:? YamlScalarNode as keyScalar), + (:? YamlScalarNode as valueScalar) -> + match keyScalar.Value.Split '-' with + | [| name; "version" |] -> + Some(name, valueScalar.Value) + | _ -> None + | _ -> None + ) + | _ -> Seq.empty + ) + | _ -> Seq.empty - inconsistentVersionsType1 || inconsistentVersionsType2 + DetectInconsistentVersionsInYamlFiles fileInfos extractVersions let DetectInconsistentVersionsInGitHubCI(dir: DirectoryInfo) = let ymlFiles = dir.GetFiles("*.yml", SearchOption.AllDirectories)