CTM-330: ordering of set values in expression evaluation

core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/CompactEntityComponent.scala

@@ -712,40 +712,41 @@ class CompactEntityQuery(driverComponent: DriverComponent)
       DBIO.successful(Map.empty[String, Seq[CompactEntityRecord]])
     } else {
       val initialPointer = EntityPointer(startingEntityType, startingEntityName)
-      val initialGrouped: Map[String, Set[EntityPointer]] =
+      val initialGrouped: Map[String, Seq[EntityPointer]] =
         if (startingEntityType == rootEntityType)
-          Map(startingEntityName -> Set(initialPointer))
+          Map(startingEntityName -> Seq(initialPointer))
         else
-          Map("" -> Set(initialPointer))
+          Map("" -> Seq(initialPointer))
 
       def traverseGroups(
-        groupedEntities: Map[String, Set[EntityPointer]],
+        groupedEntities: Map[String, Seq[EntityPointer]],
         currentEntityType: String,
         remainingChain: Seq[String],
         grouped: Boolean
-      ): ReadAction[Map[String, Set[EntityPointer]]] =
+      ): ReadAction[Map[String, Seq[EntityPointer]]] =
         if (remainingChain.isEmpty) DBIO.successful(groupedEntities)
         else {
           val relation = remainingChain.head
           // For each group, get all referenced entities for this relation
           val nextGroupsF = DBIO
             .sequence(groupedEntities.map { case (groupKey, pointers) =>
-              getEntities(workspaceId, pointers).map { entities =>
+              getEntities(workspaceId, pointers.toSet).map { entities =>
                 val nextPointers = entities.flatMap { entityRecord =>
+                  // to preserve legacy behavior, references are de-duplicated via .distinct
                   entityRecord.toEntity.attributes.get(AttributeName.fromDelimitedName(relation)) match {
                     case Some(rel: AttributeEntityReference)      => Seq(rel.toPointer)
-                    case Some(rels: AttributeEntityReferenceList) => rels.list.map(_.toPointer)
+                    case Some(rels: AttributeEntityReferenceList) => rels.list.map(_.toPointer).distinct
                     case _                                        => Seq.empty[EntityPointer]
                   }
-                }.toSet
+                }
                 groupKey -> nextPointers
               }
             }.toSeq)
             .map(_.toMap)
 
           nextGroupsF.flatMap { nextGroups =>
-            val allNextPointers = nextGroups.values.flatten.toSet
-            if (allNextPointers.map(_.entityType).size > 1) {
+            val allNextPointers = nextGroups.values.flatten.toSeq
+            if (allNextPointers.map(_.entityType).distinct.size > 1) {
               DBIO.failed(
                 new RawlsExceptionWithErrorReport(
                   ErrorReport(

core/src/test/scala/org/broadinstitute/dsde/rawls/entities/compact/CompactEntityProviderE2ESpec.scala

@@ -4,6 +4,7 @@ import akka.actor.ActorSystem
 import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.model.headers.OAuth2BearerToken
 import akka.stream.scaladsl.{Sink, Source}
+import cromwell.client.model.ValueType.TypeNameEnum
 import cromwell.client.model.{ToolInputParameter, ValueType}
 import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport
 import org.broadinstitute.dsde.rawls.dataaccess.slick.TestDriverComponentWithFlatSpecAndMatchers
@@ -27,6 +28,7 @@ import org.broadinstitute.dsde.rawls.model.{
   AttributeNumber,
   AttributeRename,
   AttributeString,
+  AttributeValueList,
   Entity,
   EntityPointer,
   RawlsRequestContext,
@@ -1288,6 +1290,169 @@ class CompactEntityProviderE2ESpec extends TestDriverComponentWithFlatSpecAndMat
 
   }
 
+  it should "return set-entity values in the set's order" in withMinimalTestDatabase { _ =>
+    // define how many set members this test should use
+    val range = Range(0, 100)
+    // define that many participants, with an "index" attribute
+    val participants = range.map { idx =>
+      Entity(s"participant_$idx", "participant", Map(AttributeName.withDefaultNS("index") -> AttributeNumber(idx)))
+    }
+    // define a participant set containing all participants
+    val participantSet = Entity("the-set",
+                                "participant_set",
+                                Map(
+                                  AttributeName.withDefaultNS("participants") -> AttributeEntityReferenceList(
+                                    participants.map(_.toReference)
+                                  )
+                                )
+    )
+
+    // save participants
+    runAndWait(
+      compactEntityQuery.batchWriteEntities(minimalTestData.workspace.workspaceIdAsUUID,
+                                            participants,
+                                            insertOnly = false
+      )
+    )
+    // save participant set
+    runAndWait(
+      compactEntityQuery.batchWriteEntities(minimalTestData.workspace.workspaceIdAsUUID,
+                                            Seq(participantSet),
+                                            insertOnly = false
+      )
+    )
+
+    // get provider
+    val provider = defaultProvider()
+
+    // set up arguments for expression evaluation
+    val expressionEvaluationContext =
+      ExpressionEvaluationContext(Option("participant_set"), Option("the-set"), None, Option("participant_set"))
+    val toolInputParameter = new ToolInputParameter()
+      .name("my-input-name")
+      .valueType(
+        new ValueType().typeName(ValueType.TypeNameEnum.ARRAY).arrayType(new ValueType().typeName(TypeNameEnum.INT))
+      )
+    val processableInputs = Set(MethodInput(toolInputParameter, "this.participants.index"))
+    val gatherInputsResult = GatherInputsResult(processableInputs, Set(), Set(), Set())
+
+    // evaluate "this.participants.index" against the participant set
+    val submissionValidationEntityInputsList =
+      Await
+        .result(provider.evaluateExpressions(expressionEvaluationContext, gatherInputsResult, Map()), atMost)
+        .toList
+    submissionValidationEntityInputsList.size shouldBe 1
+
+    // basic validation of the expression-evaluation result
+    val entityInputs = submissionValidationEntityInputsList.head
+    entityInputs.entityName shouldBe "the-set"
+    entityInputs.inputResolutions.size shouldBe 1
+    entityInputs.inputResolutions.head.error shouldBe empty
+    entityInputs.inputResolutions.head.inputName shouldBe "my-input-name"
+
+    val resolvedValue = entityInputs.inputResolutions.head.value
+    resolvedValue should not be empty
+    resolvedValue.get shouldBe a[AttributeValueList]
+
+    val actual = resolvedValue.get.asInstanceOf[AttributeValueList]
+    val expected = AttributeValueList(range.map(idx => AttributeNumber(idx)))
+
+    // did expression-evaluation return the correct attribute values, in _any_ order?
+    withClue("evaluated expression should contain the same elements, in any order") {
+      actual.list should contain theSameElementsAs expected.list
+    }
+    // did expression-evaluation return the correct attribute values, in the same order as the set members?
+    withClue("evaluated expression should contain the same elements, in the same order") {
+      actual.list should contain theSameElementsInOrderAs expected.list
+    }
+  }
+
+  it should "de-duplicate values if a set contains duplicate references" in withMinimalTestDatabase { _ =>
+    // define two participants, with an "index" attribute
+    val participants = Seq(
+      Entity(s"participant_1", "participant", Map(AttributeName.withDefaultNS("index") -> AttributeNumber(1))),
+      Entity(s"participant_2", "participant", Map(AttributeName.withDefaultNS("index") -> AttributeNumber(2))),
+      Entity(s"participant_3", "participant", Map(AttributeName.withDefaultNS("index") -> AttributeNumber(3))),
+      Entity(s"participant_4", "participant", Map(AttributeName.withDefaultNS("index") -> AttributeNumber(4)))
+    )
+
+    // define a participant set containing duplicate references to those participants
+    val ref1 = AttributeEntityReference("participant", "participant_1")
+    val ref2 = AttributeEntityReference("participant", "participant_2")
+    val ref3 = AttributeEntityReference("participant", "participant_3")
+    val ref4 = AttributeEntityReference("participant", "participant_4")
+    val participantSet = Entity(
+      "the-set",
+      "participant_set",
+      Map(
+        AttributeName.withDefaultNS("participants") -> AttributeEntityReferenceList(
+          Seq(ref1, ref2, ref1, ref3, ref2, ref4, ref2, ref3, ref1, ref4)
+        )
+      )
+    )
+
+    // save participants
+    runAndWait(
+      compactEntityQuery.batchWriteEntities(minimalTestData.workspace.workspaceIdAsUUID,
+                                            participants,
+                                            insertOnly = false
+      )
+    )
+    // save participant set
+    runAndWait(
+      compactEntityQuery.batchWriteEntities(minimalTestData.workspace.workspaceIdAsUUID,
+                                            Seq(participantSet),
+                                            insertOnly = false
+      )
+    )
+
+    // get provider
+    val provider = defaultProvider()
+
+    // set up arguments for expression evaluation
+    val expressionEvaluationContext =
+      ExpressionEvaluationContext(Option("participant_set"), Option("the-set"), None, Option("participant_set"))
+    val toolInputParameter = new ToolInputParameter()
+      .name("my-input-name")
+      .valueType(
+        new ValueType().typeName(ValueType.TypeNameEnum.ARRAY).arrayType(new ValueType().typeName(TypeNameEnum.INT))
+      )
+    val processableInputs = Set(MethodInput(toolInputParameter, "this.participants.index"))
+    val gatherInputsResult = GatherInputsResult(processableInputs, Set(), Set(), Set())
+
+    // evaluate "this.participants.index" against the participant set
+    val submissionValidationEntityInputsList =
+      Await
+        .result(provider.evaluateExpressions(expressionEvaluationContext, gatherInputsResult, Map()), atMost)
+        .toList
+    submissionValidationEntityInputsList.size shouldBe 1
+
+    // basic validation of the expression-evaluation result
+    val entityInputs = submissionValidationEntityInputsList.head
+    entityInputs.entityName shouldBe "the-set"
+    entityInputs.inputResolutions.size shouldBe 1
+    entityInputs.inputResolutions.head.error shouldBe empty
+    entityInputs.inputResolutions.head.inputName shouldBe "my-input-name"
+
+    val resolvedValue = entityInputs.inputResolutions.head.value
+    resolvedValue should not be empty
+    resolvedValue.get shouldBe a[AttributeValueList]
+
+    val actual = resolvedValue.get.asInstanceOf[AttributeValueList]
+    val expected = AttributeValueList(
+      Seq(AttributeNumber(1), AttributeNumber(2), AttributeNumber(3), AttributeNumber(4))
+    )
+
+    // did expression-evaluation return the correct attribute values, in _any_ order?
+    withClue("evaluated expression should contain the same elements, in any order") {
+      actual.list should contain theSameElementsAs expected.list
+    }
+    // did expression-evaluation return the correct attribute values, in the same order as the set members?
+    withClue("evaluated expression should contain the same elements, in the same order") {
+      actual.list should contain theSameElementsInOrderAs expected.list
+    }
+  }
+
   // ====================================================================================================
   //  helper methods
   // ====================================================================================================