Merge branch 'develop' into feature/improve-analytics-logs

.github/workflows/unit-test.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - master
       - develop
+  workflow_dispatch:
 
 jobs:
   unit-test:

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/mute/muterunner.kt

@@ -40,6 +40,7 @@ import com.smeup.rpgparser.parsing.parsetreetoast.toAst
 import com.smeup.rpgparser.rpginterop.DirRpgProgramFinder
 import com.smeup.rpgparser.rpginterop.RpgProgramFinder
 import com.smeup.rpgparser.utils.asDouble
+import com.smeup.rpgparser.utils.popIfPresent
 import com.strumenta.kolasu.validation.Error
 import java.io.*
 import java.nio.file.Files
@@ -242,7 +243,7 @@ fun RpgParserResult.executeMuteAnnotations(
     interpreter.execute(cu, parameters)
     interpreter.doSomethingAfterExecution()
     mainExecutionContext?.let {
-        it.parsingProgramStack.pop()
+        it.parsingProgramStack.popIfPresent()
         it.programStack.pop()
     }
     return interpreter.getSystemInterface().executedAnnotationInternal.toSortedMap()

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/parsing/facade/RpgParserFacade.kt

@@ -39,6 +39,8 @@ import com.smeup.rpgparser.parsing.parsetreetoast.setOverlayOn
 import com.smeup.rpgparser.parsing.parsetreetoast.toAst
 import com.smeup.rpgparser.utils.insLineNumber
 import com.smeup.rpgparser.utils.parseTreeToXml
+import com.smeup.rpgparser.utils.popIfPresent
+import com.smeup.rpgparser.utils.pushIfNotAlreadyPresent
 import com.strumenta.kolasu.model.Point
 import com.strumenta.kolasu.model.Position
 import com.strumenta.kolasu.model.endPoint
@@ -454,7 +456,7 @@ class RpgParserFacade {
         inputStream: InputStream,
         sourceProgram: SourceProgram? = SourceProgram.RPGLE
     ): CompilationUnit {
-        MainExecutionContext.getParsingProgramStack().push(ParsingProgram(executionProgramName))
+        MainExecutionContext.getParsingProgramStack().pushIfNotAlreadyPresent(ParsingProgram(executionProgramName))
         val cu = if (sourceProgram?.sourceType == true) {
             (tryToLoadCompilationUnit() ?: createAst(inputStream)).apply {
                 MainExecutionContext.getConfiguration().jarikoCallback.afterAstCreation.invoke(this)
@@ -466,7 +468,10 @@ class RpgParserFacade {
             }
         }
         // This is a trick to pass the popped ParsingProgramStack for further processing
-        addLastPoppedParsingProgram(MainExecutionContext.getParsingProgramStack().pop())
+        MainExecutionContext.getParsingProgramStack().popIfPresent()?.apply {
+            addLastPoppedParsingProgram(this)
+        }
+
         return cu
     }
 

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/parsing/parsetreetoast/misc.kt

@@ -104,13 +104,17 @@ private fun RContext.getDataDefinitions(
     fileDefinitions.values.flatten().toList().removeDuplicatedDataDefinition().forEach {
         dataDefinitionProviders.add(it.updateKnownDataDefinitionsAndGetHolder(knownDataDefinitions))
     }
+
+    // Move the D specs with like because depending on other D specs definitions
+    val sortedStatements = this.statement().moveLikeStatementToTheEnd(conf = conf)
+
     // First pass ignore exception and all the know definitions
-    dataDefinitionProviders.addAll(this.statement()
+    dataDefinitionProviders.addAll(sortedStatements
         .mapNotNull {
             it.toDataDefinitionProvider(conf = conf, knownDataDefinitions = knownDataDefinitions)
         })
     // Second pass, everything, I mean everything
-    dataDefinitionProviders.addAll(this.statement()
+    dataDefinitionProviders.addAll(sortedStatements
         .mapNotNull {
             kotlin.runCatching {
                 when {
@@ -125,15 +129,22 @@ private fun RContext.getDataDefinitions(
                             .updateKnownDataDefinitionsAndGetHolder(knownDataDefinitions)
                     }
                     it.dcl_ds() != null && it.dcl_ds().useLikeDs(conf) -> {
-                        DataDefinitionCalculator(it.dcl_ds().toAstWithLikeDs(
-                            conf = conf,
-                            dataDefinitionProviders = dataDefinitionProviders)
-                        )
+                        DataDefinitionCalculator(
+                            it.dcl_ds().toAstWithLikeDs(
+                                conf = conf,
+                                dataDefinitionProviders = dataDefinitionProviders
+                            )
+                        ).toDataDefinition().updateKnownDataDefinitionsAndGetHolder(knownDataDefinitions)
                     }
                     it.dcl_ds() != null && it.dcl_ds().useExtName() && fileDefinitions.keys.any { fileDefinition ->
                         fileDefinition.name.equals(it.dcl_ds().getKeywordExtName().getExtName(), ignoreCase = true)
                     } -> {
-                        DataDefinitionCalculator(it.dcl_ds().toAstWithExtName(conf, fileDefinitions))
+                        DataDefinitionCalculator(
+                            it.dcl_ds().toAstWithExtName(
+                                conf = conf,
+                                fileDefinitions = fileDefinitions
+                            )
+                        ).toDataDefinition().updateKnownDataDefinitionsAndGetHolder(knownDataDefinitions)
                     }
                     else -> null
                 }
@@ -143,6 +154,16 @@ private fun RContext.getDataDefinitions(
     return dataDefinitionProviders.mapNotNull { kotlin.runCatching { it.toDataDefinition() }.getOrNull() }
 }
 
+private fun List<StatementContext>.moveLikeStatementToTheEnd(conf: ToAstConfiguration): List<StatementContext> {
+    val likeStatements = this.filter {
+        it.dcl_ds()?.useLikeDs(conf = conf) ?: it.dspec()?.useLike() ?: false
+    }
+    val otherStatements = this.filter {
+        !(it.dcl_ds()?.useLikeDs(conf = conf) ?: it.dspec()?.useLike() ?: false)
+    }
+    return otherStatements + likeStatements
+}
+
 private fun DataDefinition.updateKnownDataDefinitionsAndGetHolder(
     knownDataDefinitions: MutableMap<String, DataDefinition>
 ): DataDefinitionHolder {
@@ -151,10 +172,13 @@ private fun DataDefinition.updateKnownDataDefinitionsAndGetHolder(
 }
 
 private fun MutableMap<String, DataDefinition>.addIfNotPresent(dataDefinition: DataDefinition) {
-    if (put(dataDefinition.name, dataDefinition) != null)
+    val old = get(dataDefinition.name)
+    if (old == null || (old.type is RecordFormatType && dataDefinition.type is DataStructureType)) {
+        put(dataDefinition.name, dataDefinition)
+    } else {
         dataDefinition.error("${dataDefinition.name} has been defined twice")
+    }
 }
-
 internal fun FileDefinition.toDataDefinitions(): List<DataDefinition> {
     val dataDefinitions = mutableListOf<DataDefinition>()
     val reloadConfig = MainExecutionContext.getConfiguration()
@@ -371,6 +395,10 @@ private fun Dcl_dsContext.useLikeDs(conf: ToAstConfiguration): Boolean {
     return (this.keyword().any { it.keyword_likeds() != null })
 }
 
+private fun DspecContext.useLike(): Boolean {
+    return this.keyword().any { it.keyword_like() != null }
+}
+
 private fun Dcl_dsContext.useExtName(): Boolean {
     return this.keyword().any { it.keyword_extname() != null }
 }
@@ -1417,7 +1445,9 @@ private fun FactorContext.toIndexedExpression(conf: ToAstConfiguration): Pair<Ex
     if (this.text.contains(":")) this.text.toIndexedExpression(toPosition(conf.considerPosition)) else this.content.toAst(conf) to null
 
 private fun String.toIndexedExpression(position: Position?): Pair<Expression, Int?> {
-    val baseStringTokens = this.split(":")
+    val quoteAwareSplitPattern = Regex(""":(?=([^']*'[^']*')*[^']*$)""")
+    val baseStringTokens = this.split(quoteAwareSplitPattern)
+
     val startPosition =
         when (baseStringTokens.size) {
             !in 1..2 -> throw UnsupportedOperationException("Wrong base string expression at line ${position?.line()}: $this")

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/parsing/parsetreetoast/resolution.kt

@@ -24,7 +24,10 @@ import com.smeup.rpgparser.interpreter.InStatementDataDefinition
 import com.smeup.rpgparser.interpreter.type
 import com.smeup.rpgparser.parsing.ast.*
 import com.smeup.rpgparser.parsing.facade.AstCreatingException
+import com.smeup.rpgparser.parsing.facade.getExecutionProgramNameWithNoExtension
 import com.smeup.rpgparser.parsing.facade.getLastPoppedParsingProgram
+import com.smeup.rpgparser.utils.popIfPresent
+import com.smeup.rpgparser.utils.pushIfNotAlreadyPresent
 import com.strumenta.kolasu.model.*
 import com.strumenta.kolasu.validation.Error
 import com.strumenta.kolasu.validation.ErrorType
@@ -119,14 +122,14 @@ fun MuteAnnotation.resolveAndValidate(cu: CompilationUnit) {
  */
 fun CompilationUnit.resolveAndValidate(): List<Error> {
     kotlin.runCatching {
-        val parsingProgram = ParsingProgram(MainExecutionContext.getExecutionProgramName())
+        val parsingProgram = ParsingProgram(getExecutionProgramNameWithNoExtension())
         parsingProgram.copyBlocks = this.copyBlocks
         parsingProgram.sourceLines = getLastPoppedParsingProgram()?.sourceLines
-        MainExecutionContext.getParsingProgramStack().push(parsingProgram)
+        MainExecutionContext.getParsingProgramStack().pushIfNotAlreadyPresent(parsingProgram)
         this.resolve()
         checkAstCreationErrors(AstHandlingPhase.Resolution)
         return this.validate().apply {
-            MainExecutionContext.getParsingProgramStack().pop()
+            MainExecutionContext.getParsingProgramStack().popIfPresent()
         }
     }.onFailure {
         this.source?.let { source ->

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/parsing/parsetreetoast/statements.kt

@@ -123,12 +123,10 @@ internal fun RpgParser.CsDOUxxContext.toAst(blockContext: BlockContext, conf: To
         else -> todo(conf = conf)
     }
 
-    val factor2Ast = factor2.toAstIfSymbolicConstant() ?: factor2.content.toAst(conf)
-
     return DOUxxStmt(
         comparisonOperator = comparison,
-        factor1 = this.factor1.content.toAst(conf = conf),
-        factor2 = factor2Ast,
+        factor1 = factor1.toAstIfSymbolicConstant() ?: factor1.content?.toAst(conf) ?: factor1.error("Factor 1 cannot be null", conf = conf),
+        factor2 = factor2.toAstIfSymbolicConstant() ?: factor2.content?.toAst(conf) ?: factor1.error("Factor 2 cannot be null", conf = conf),
         position = toPosition(conf.considerPosition),
         body = blockContext.statement().map { it.toAst(conf) }
     )

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/utils/misc.kt

@@ -16,7 +16,9 @@
 
 package com.smeup.rpgparser.utils
 
+import com.smeup.rpgparser.execution.ParsingProgram
 import java.math.BigDecimal
+import java.util.*
 import kotlin.system.measureTimeMillis
 
 fun measureAndPrint(block: () -> Unit) {
@@ -132,4 +134,26 @@ fun String.insLineNumber(padChars: Int, filter: (lineNumber: Int) -> Boolean): S
         }
         sb.toString()
     }
+}
+
+/**
+ * Push parsingProgram in a stack if it is not already present
+ * @param parsingProgram The element to push
+ * */
+internal fun Stack<ParsingProgram>.pushIfNotAlreadyPresent(parsingProgram: ParsingProgram) {
+    if (this.isEmpty() || this.peek() != parsingProgram) {
+        this.push(parsingProgram)
+    }
+}
+
+/**
+ * Pop a ParsingProgram from a stack if it is present
+ * @return The element popped or null if the stack is empty
+ * */
+internal fun Stack<ParsingProgram>.popIfPresent(): ParsingProgram? {
+    return if (this.isNotEmpty()) {
+        this.pop()
+    } else {
+        null
+    }
 }

rpgJavaInterpreter-core/src/test/kotlin/com/smeup/rpgparser/interpreter/JarikoCallbackTest.kt

@@ -24,6 +24,7 @@ import com.smeup.rpgparser.parsing.facade.Copy
 import com.smeup.rpgparser.parsing.facade.CopyId
 import com.smeup.rpgparser.parsing.facade.SourceReference
 import com.smeup.rpgparser.parsing.facade.SourceReferenceType
+import com.smeup.rpgparser.parsing.parsetreetoast.ParseTreeToAstError
 import com.smeup.rpgparser.parsing.parsetreetoast.ToAstConfiguration
 import org.junit.Assert
 import java.io.StringReader
@@ -513,6 +514,19 @@ class JarikoCallbackTest : AbstractTest() {
         executeSourceLineTest("ERROR22")
     }
 
+    @Test
+    fun executeERROR23CallBackTest() {
+        executePgmCallBackTest("ERROR23", SourceReferenceType.Program, "ERROR23", mapOf(
+            9 to "Factor 2 cannot be null",
+            14 to "Factor 1 cannot be null"
+        ))
+    }
+
+    @Test
+    fun executeERROR23SourceLineTest() {
+        executeSourceLineTest("ERROR23")
+    }
+
     @Test
     fun bypassSyntaxErrorTest() {
         val configuration = Configuration().apply {
@@ -637,6 +651,53 @@ class JarikoCallbackTest : AbstractTest() {
         }
     }
 
+    /**
+     * This function is used to test the execution of a program and validate the error handling mechanism.
+     * It expects the program to fail and checks if the error events are correctly captured.
+     *
+     * @param pgm The name of the program to be executed.
+     * @param sourceReferenceType The expected type of the source reference (Program or Copy) where the error is expected to occur.
+     * @param sourceId The expected identifier of the source where the error is expected to occur.
+     * @param lines The map of lines, number and message, expected.
+     */
+    private fun executePgmCallBackTest(pgm: String, sourceReferenceType: SourceReferenceType, sourceId: String, lines: Map<Int, String>) {
+        val errorEvents = mutableListOf<ErrorEvent>()
+        runCatching {
+            val configuration = Configuration().apply {
+                jarikoCallback.onError = { errorEvent ->
+                    println(errorEvent)
+                    errorEvents.add(errorEvent)
+                }
+                options = Options(debuggingInformation = true)
+                reloadConfig = createMockReloadConfig()
+            }
+            executePgm(pgm, configuration = configuration)
+        }.onSuccess {
+            Assert.fail("Program must exit with error")
+        }.onFailure {
+            println(it.stackTraceToString())
+            Assert.assertEquals(sourceReferenceType, errorEvents[0].sourceReference!!.sourceReferenceType)
+            Assert.assertEquals(sourceId, errorEvents[0].sourceReference!!.sourceId)
+            val found = errorEvents
+                            .associate { errorEvent -> errorEvent.sourceReference!!.relativeLine to (errorEvent.error as ParseTreeToAstError).message!! }
+                            .map { it.contains(lines) }
+            Assert.assertTrue(
+                "Errors don't correspond",
+                found.filter { it }.size.equals(lines.size)
+            )
+        }
+    }
+
+    internal fun Map.Entry<Int, String>.contains(list: Map<Int, String>): Boolean {
+        list.forEach {
+            if (this.value.contains(it.value) && this.key == it.key) {
+                return true
+            }
+        }
+
+        return false
+    }
+
     /**
      * Verify that the sourceLine is properly set in case of error.
      * ErrorEvent must contain a reference of an absolute line of the source code

rpgJavaInterpreter-core/src/test/kotlin/com/smeup/rpgparser/parsing/ast/ToAstSmokeTest.kt

@@ -66,16 +66,17 @@ open class ToAstSmokeTest : AbstractTest() {
     fun buildAstForJD_001_dataDefinitions() {
         val root = assertASTCanBeProduced("JD_001")
         assertEquals(10, root.dataDefinitions.size)
-        assertEquals("@UNNAMED_DS_16", root.dataDefinitions[0].name)
-        assertEquals("U\$FUNZ", root.dataDefinitions[1].name)
-        assertEquals("U\$METO", root.dataDefinitions[2].name)
-        assertEquals("U\$SVARSK", root.dataDefinitions[3].name)
-        assertEquals("U\$IN35", root.dataDefinitions[4].name)
-        assertEquals("\$\$URL", root.dataDefinitions[5].name)
-        assertEquals("\$X", root.dataDefinitions[6].name)
-        assertEquals("U\$SVARSK_INI", root.dataDefinitions[7].name)
-        assertEquals("§§FUNZ", root.dataDefinitions[8].name)
-        assertEquals("§§METO", root.dataDefinitions[9].name)
+        val dataDefinitionNames = root.dataDefinitions.map { it.name }
+        assertTrue(dataDefinitionNames.contains("@UNNAMED_DS_16"))
+        assertTrue(dataDefinitionNames.contains("U\$FUNZ"))
+        assertTrue(dataDefinitionNames.contains("U\$METO"))
+        assertTrue(dataDefinitionNames.contains("U\$SVARSK"))
+        assertTrue(dataDefinitionNames.contains("U\$IN35"))
+        assertTrue(dataDefinitionNames.contains("\$\$URL"))
+        assertTrue(dataDefinitionNames.contains("\$X"))
+        assertTrue(dataDefinitionNames.contains("U\$SVARSK_INI"))
+        assertTrue(dataDefinitionNames.contains("§§FUNZ"))
+        assertTrue(dataDefinitionNames.contains("§§METO"))
     }
 
     @Test

rpgJavaInterpreter-core/src/test/kotlin/com/smeup/rpgparser/smeup/MULANGT02ConstAndDSpecTest.kt

@@ -231,4 +231,13 @@ open class MULANGT02ConstAndDSpecTest : MULANGTTest() {
         val expected = listOf("Size: 8")
         assertEquals(expected, "smeup/MU024013".outputOf(configuration = smeupConfig))
     }
+
+    @Test
+    fun executeMUDRNRAPU00202() {
+        MULANGTLDbMock().use {
+            com.smeup.rpgparser.db.utilities.execute(listOf(it.createTable(), it.populateTable()))
+            val expected = listOf("ok")
+            assertEquals(expected, "smeup/MUDRNRAPU00202".outputOf(configuration = smeupConfig))
+        }
+    }
 }

rpgJavaInterpreter-core/src/test/kotlin/com/smeup/rpgparser/smeup/MULANGT10BaseCodopTest.kt

@@ -74,6 +74,16 @@ open class MULANGT10BaseCodopTest : MULANGTTest() {
         assertEquals(expected, "smeup/T10_A60_P10".outputOf())
     }
 
+    /**
+     * SCAN with special char
+     * @see LS24002777
+     */
+    @Test
+    fun executeMU103511() {
+        val expected = listOf("Found at: 3")
+        assertEquals(expected, "smeup/MU103511".outputOf())
+    }
+
     /**
      * Utilization of `LIKEDS` with a `DataDefinition` defined in parent.
      * @see #271

rpgJavaInterpreter-core/src/test/kotlin/com/smeup/rpgparser/smeup/MULANGT12LoopFlowGotoTest.kt

@@ -96,4 +96,13 @@ open class MULANGT12LoopFlowGotoTest : MULANGTTest() {
         val expected = listOf("A09_N2: 10")
         assertEquals(expected, "smeup/MU120906".outputOf())
     }
+
+    /**
+     * DOWEQ with indicator into SR
+     */
+    @Test
+    fun executeMU120907() {
+        val expected = listOf("HELLO THERE")
+        assertEquals(expected, "smeup/MU120907".outputOf())
+    }
 }