Merge PR #177 from webwarrior-ws/inconsistent-versions-env-var

Examine env. var contents in GithubCI files for version
consistency check.

scripts/eofConvention.fsx

@@ -4,6 +4,7 @@ open System.IO
 open System
 
 #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1"
+#r "nuget: YamlDotNet, Version=16.1.3"
 
 #load "../src/FileConventions/Helpers.fs"
 #load "../src/FileConventions/Library.fs"

scripts/executableConvention.fsx

@@ -4,6 +4,7 @@ open System
 open System.IO
 
 #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1"
+#r "nuget: YamlDotNet, Version=16.1.3"
 #load "../src/FileConventions/Library.fs"
 #load "../src/FileConventions/Helpers.fs"
 

scripts/inconsistentVersionsInFSharpScripts.fsx

@@ -4,6 +4,7 @@ open System.IO
 open System.Linq
 
 #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1"
+#r "nuget: YamlDotNet, Version=16.1.3"
 
 #load "../src/FileConventions/Library.fs"
 #load "../src/FileConventions/Helpers.fs"

scripts/inconsistentVersionsInGitHubCI.fsx

@@ -3,6 +3,7 @@
 open System.IO
 
 #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1"
+#r "nuget: YamlDotNet, Version=16.1.3"
 
 #load "../src/FileConventions/Library.fs"
 #load "../src/FileConventions/Helpers.fs"

scripts/mixedLineEndings.fsx

@@ -4,6 +4,7 @@ open System
 open System.IO
 
 #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1"
+#r "nuget: YamlDotNet, Version=16.1.3"
 
 #load "../src/FileConventions/Library.fs"
 #load "../src/FileConventions/Helpers.fs"

scripts/nonVerboseFlagsInGitHubCIAndScripts.fsx

@@ -4,6 +4,7 @@ open System
 open System.IO
 
 #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1"
+#r "nuget: YamlDotNet, Version=16.1.3"
 
 #load "../src/FileConventions/Library.fs"
 #load "../src/FileConventions/Helpers.fs"

scripts/shebangConvention.fsx

@@ -4,6 +4,7 @@ open System
 open System.IO
 
 #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1"
+#r "nuget: YamlDotNet, Version=16.1.3"
 
 #load "../src/FileConventions/Library.fs"
 #load "../src/FileConventions/Helpers.fs"

scripts/unpinnedDotnetToolInstallVersions.fsx

@@ -4,6 +4,7 @@ open System
 open System.IO
 
 #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1"
+#r "nuget: YamlDotNet, Version=16.1.3"
 
 #load "../src/FileConventions/Library.fs"
 #load "../src/FileConventions/Helpers.fs"

scripts/unpinnedGitHubActionsImageVersions.fsx

@@ -4,6 +4,7 @@ open System
 open System.IO
 
 #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1"
+#r "nuget: YamlDotNet, Version=16.1.3"
 
 #load "../src/FileConventions/Library.fs"
 #load "../src/FileConventions/Helpers.fs"

scripts/unpinnedNugetPackageReferenceVersionsInFSharpScripts.fsx

@@ -4,6 +4,7 @@ open System
 open System.IO
 
 #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1"
+#r "nuget: YamlDotNet, Version=16.1.3"
 
 #load "../src/FileConventions/Library.fs"
 #load "../src/FileConventions/Helpers.fs"

scripts/unpinnedNugetPackageReferenceVersionsInProjects.fsx

@@ -4,6 +4,7 @@ open System
 open System.IO
 
 #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1"
+#r "nuget: YamlDotNet, Version=16.1.3"
 
 #load "../src/FileConventions/Library.fs"
 #load "../src/FileConventions/Helpers.fs"

scripts/wrapLatestCommitMsg.fsx

@@ -6,6 +6,7 @@ open System.Text.RegularExpressions
 open System.Linq
 
 #r "nuget: Mono.Unix, Version=7.1.0-final.1.21458.1"
+#r "nuget: YamlDotNet, Version=16.1.3"
 
 #load "../src/FileConventions/Library.fs"
 

src/FileConventions.Test/DummyFiles/DummyWorkflowsWithEnv/DummyCIWithPulumiVersionInEnvV2.0.0.yml

@@ -0,0 +1,17 @@
+name: CI
+
+on: [push, pull_request]
+
+env:
+  PULUMI_VERSION: 2.0.0
+
+jobs:
+  jobA:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Pulumi CLI
+        with:
+          pulumi-version: ${{ PULUMI_VERSION }}
+      - name: Print "Hello World!"
+        run: echo "Hello World!"

src/FileConventions.Test/DummyFiles/DummyWorkflowsWithEnv/DummyCIWithPulumiVersionInEnvV2.0.1.yml

@@ -0,0 +1,17 @@
+name: CI
+
+on: [push, pull_request]
+
+env:
+  PULUMI_VERSION: 2.0.1
+
+jobs:
+  jobA:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Pulumi CLI
+        with:
+          pulumi-version: ${{ PULUMI_VERSION }}
+      - name: Print "Hello World!"
+        run: echo "Hello World!"

src/FileConventions.Test/FileConventions.Test.fs

@@ -420,6 +420,14 @@ let DetectInconsistentVersionsInGitHubCI1() =
 
     Assert.That(DetectInconsistentVersionsInGitHubCI fileInfo, Is.EqualTo true)
 
+[<Test>]
+let DetectInconsistentVersionsInGitHubCI2() =
+    let fileInfo =
+        DirectoryInfo(
+            Path.Combine(dummyFilesDirectory.FullName, "DummyWorkflowsWithEnv")
+        )
+
+    Assert.That(DetectInconsistentVersionsInGitHubCI fileInfo, Is.EqualTo true)
 
 [<Test>]
 let DetectInconsistentVersionsInNugetRefsInFSharpScripts1() =

src/FileConventions/FileConventions.fsproj

@@ -13,6 +13,7 @@
   <ItemGroup>
     <PackageReference Include="Mono.Posix" Version="7.1.0-final.1.21458.1" />
     <PackageReference Include="Mono.Unix" Version="7.1.0-final.1.21458.1" />
+    <PackageReference Include="YamlDotNet" Version="16.1.3" />
   </ItemGroup>
 
 </Project>

src/FileConventions/Library.fs

@@ -7,6 +7,7 @@ open System.Text.RegularExpressions
 
 open Mono
 open Mono.Unix.Native
+open YamlDotNet.RepresentationModel
 
 let SplitByEOLs (text: string) (numberOfEOLs: uint) =
     if numberOfEOLs = 0u then
@@ -300,21 +301,105 @@ let private DetectInconsistentVersions
     |> Seq.map(fun item -> Seq.length item.Value > 1)
     |> Seq.contains true
 
+let private DetectInconsistentVersionsInYamlFiles
+    (fileInfos: seq<FileInfo>)
+    (extractVersionsFunction: YamlNode -> seq<string * string>)
+    =
+    let envVarRegex =
+        Regex(@"\s*\$\{\{\s*([^\s\}]+)\s*\}\}\s*", RegexOptions.Compiled)
+
+    let yamlDocuments =
+        Seq.map
+            (fun (fileInfo: FileInfo) ->
+                let yaml = YamlStream()
+                use reader = new StreamReader(fileInfo.FullName)
+                yaml.Load reader
+                yaml.Documents[0].RootNode
+            )
+            fileInfos
+
+    let versionMap =
+        Seq.fold
+            (fun mapping (yamlDoc: YamlNode) ->
+                let matches =
+                    Seq.collect extractVersionsFunction yamlDoc.AllNodes
+
+                matches
+                |> Seq.fold
+                    (fun acc (key, value) ->
+                        let actualValue =
+                            let variableRegexMatch = envVarRegex.Match value
+
+                            if variableRegexMatch.Success then
+                                let yamlDict = yamlDoc :?> YamlMappingNode
+
+                                let envDict =
+                                    yamlDict.Children.["env"]
+                                    :?> YamlMappingNode
+
+                                let envVarName =
+                                    variableRegexMatch.Groups.[1].Value
+
+                                match envDict.Children.TryGetValue envVarName
+                                    with
+                                | true, envVarValue ->
+                                    (envVarValue :?> YamlScalarNode).Value
+                                | false, _ ->
+                                    failwithf "env. var %s not found" envVarName
+                            else
+                                value
+
+                        match Map.tryFind key acc with
+                        | Some prevSet ->
+                            Map.add key (Set.add actualValue prevSet) acc
+                        | None -> Map.add key (Set.singleton actualValue) acc
+                    )
+                    mapping
+            )
+            Map.empty
+            yamlDocuments
+
+    versionMap
+    |> Seq.map(fun item -> Seq.length item.Value > 1)
+    |> Seq.contains true
+
 let DetectInconsistentVersionsInGitHubCIWorkflow(fileInfos: seq<FileInfo>) =
     fileInfos
     |> Seq.iter(fun fileInfo -> assert (fileInfo.FullName.EndsWith ".yml"))
 
-    let inconsistentVersionsType1 =
-        DetectInconsistentVersions
-            fileInfos
-            "\\swith:\\s*([^\\s]*)-version:\\s*([^\\s]*)\\s"
-
-    let inconsistentVersionsType2 =
-        DetectInconsistentVersions
-            fileInfos
-            "\\suses:\\s*([^\\s]*)@v([^\\s]*)\\s"
+    let extractVersions(node: YamlNode) =
+        match node with
+        | :? YamlMappingNode as yamlDict ->
+            yamlDict.Children
+            |> Seq.collect(fun (KeyValue(keyNode, valueNode)) ->
+                match keyNode, valueNode with
+                | (:? YamlScalarNode as keyScalar),
+                  (:? YamlScalarNode as valueScalar) when
+                    keyScalar.Value = "uses"
+                    ->
+                    match valueScalar.Value.Split "@v" with
+                    | [| name; version |] -> Seq.singleton(name, version)
+                    | _ -> Seq.empty
+                | (:? YamlScalarNode as keyScalar),
+                  (:? YamlMappingNode as valueMapping) when
+                    keyScalar.Value = "with"
+                    ->
+                    valueMapping.Children
+                    |> Seq.choose(fun (KeyValue(innerKeyNode, innerValueNode)) ->
+                        match innerKeyNode, innerValueNode with
+                        | (:? YamlScalarNode as keyScalar),
+                          (:? YamlScalarNode as valueScalar) ->
+                            match keyScalar.Value.Split '-' with
+                            | [| name; "version" |] ->
+                                Some(name, valueScalar.Value)
+                            | _ -> None
+                        | _ -> None
+                    )
+                | _ -> Seq.empty
+            )
+        | _ -> Seq.empty
 
-    inconsistentVersionsType1 || inconsistentVersionsType2
+    DetectInconsistentVersionsInYamlFiles fileInfos extractVersions
 
 let DetectInconsistentVersionsInGitHubCI(dir: DirectoryInfo) =
     let ymlFiles = dir.GetFiles("*.yml", SearchOption.AllDirectories)