From c582b79787eb32df2c6615566b75eabe9ad66c77 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 10 Jul 2024 13:05:09 +0200 Subject: [PATCH 01/31] Update fa-db and associated imports --- project/Dependencies.scala | 4 ++-- .../absa/atum/server/api/controller/BaseController.scala | 2 +- .../server/api/database/PostgresDatabaseProvider.scala | 2 +- .../za/co/absa/atum/server/api/database/flows/Flows.scala | 4 ++-- .../api/database/flows/functions/GetFlowCheckpoints.scala | 6 +++--- .../za/co/absa/atum/server/api/database/runs/Runs.scala | 4 ++-- .../runs/functions/CreateOrUpdateAdditionalData.scala | 8 ++++---- .../runs/functions/CreatePartitioningIfNotExists.scala | 8 ++++---- .../runs/functions/GetPartitioningAdditionalData.scala | 6 +++--- .../runs/functions/GetPartitioningCheckpoints.scala | 6 +++--- .../database/runs/functions/GetPartitioningMeasures.scala | 6 +++--- .../api/database/runs/functions/WriteCheckpoint.scala | 8 ++++---- .../absa/atum/server/api/repository/BaseRepository.scala | 2 +- .../atum/server/api/repository/CheckpointRepository.scala | 2 +- .../server/api/repository/CheckpointRepositoryImpl.scala | 2 +- .../server/api/repository/PartitioningRepository.scala | 2 +- .../api/repository/PartitioningRepositoryImpl.scala | 2 +- .../za/co/absa/atum/server/api/service/BaseService.scala | 2 +- .../absa/atum/server/api/service/CheckpointService.scala | 2 +- .../atum/server/api/service/CheckpointServiceImpl.scala | 2 +- .../atum/server/api/service/PartitioningService.scala | 2 +- .../atum/server/api/service/PartitioningServiceImpl.scala | 2 +- .../api/controller/CheckpointControllerUnitTests.scala | 4 ++-- .../CreateOrUpdateAdditionalDataIntegrationTests.scala | 5 +++-- .../runs/functions/WriteCheckpointIntegrationTests.scala | 4 ++-- .../api/repository/CheckpointRepositoryUnitTests.scala | 4 ++-- .../api/repository/PartitioningRepositoryUnitTests.scala | 4 ++-- .../server/api/service/CheckpointServiceUnitTests.scala | 4 ++-- .../server/api/service/PartitioningServiceUnitTests.scala | 4 ++-- 29 files changed, 57 insertions(+), 56 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index dfc97afe1..ee4790e6a 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -45,7 +45,7 @@ object Dependencies { val postgresql = "42.6.0" - val fadb = "0.3.0" + val fadb = "0.4.1" val json4s_spark2 = "3.5.3" val json4s_spark3 = "3.7.0-M11" @@ -134,7 +134,7 @@ object Dependencies { val zioOrg = "dev.zio" val tapirOrg = "com.softwaremill.sttp.tapir" val http4sOrg = "org.http4s" - val faDbOrg = "za.co.absa.fa-db" + val faDbOrg = "za.co.absa.db.fa-db" val playOrg = "org.playframework" val sbtOrg = "com.github.sbt" val logbackOrg = "ch.qos.logback" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index 1877cc0b4..7f290c97f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.model.ErrorResponse.{ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} -import za.co.absa.fadb.exceptions.StatusException +import za.co.absa.db.fadb.exceptions.StatusException import zio._ trait BaseController { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/PostgresDatabaseProvider.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/PostgresDatabaseProvider.scala index 68ab05c56..27d1bdf87 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/PostgresDatabaseProvider.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/PostgresDatabaseProvider.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.database import doobie.Transactor -import za.co.absa.fadb.doobie.DoobieEngine +import za.co.absa.db.fadb.doobie.DoobieEngine import zio._ import zio.interop.catz._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/Flows.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/Flows.scala index 94ca4c13e..4e993a7c3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/Flows.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/Flows.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.database.flows -import za.co.absa.fadb.DBSchema -import za.co.absa.fadb.naming.implementations.SnakeCaseNaming.Implicits._ +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.naming.implementations.SnakeCaseNaming.Implicits._ object Flows extends DBSchema diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 0b30ee8d3..723a0530f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -24,9 +24,9 @@ import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.flows.Flows import za.co.absa.atum.server.model.{CheckpointFromDB, PartitioningForDB} -import za.co.absa.fadb.DBSchema -import za.co.absa.fadb.doobie.DoobieEngine -import za.co.absa.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.doobie.DoobieEngine +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction import zio._ import zio.interop.catz._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/Runs.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/Runs.scala index 13b9fd807..c950cbf54 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/Runs.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/Runs.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.database.runs -import za.co.absa.fadb.DBSchema -import za.co.absa.fadb.naming.implementations.SnakeCaseNaming.Implicits._ +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.naming.implementations.SnakeCaseNaming.Implicits._ object Runs extends DBSchema diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index 563d8ace3..0a0e1babe 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -24,10 +24,10 @@ import za.co.absa.atum.model.dto.AdditionalDataSubmitDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import za.co.absa.atum.server.model.PartitioningForDB -import za.co.absa.fadb.DBSchema -import za.co.absa.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus -import za.co.absa.fadb.doobie.{DoobieEngine, StatusWithData} -import za.co.absa.fadb.status.handling.implementations.StandardStatusHandling +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus +import za.co.absa.db.fadb.doobie.{DoobieEngine, StatusWithData} +import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import zio._ import zio.interop.catz._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 8c6135476..7a83a0031 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -22,10 +22,10 @@ import doobie.util.Read import play.api.libs.json.Json import za.co.absa.atum.model.dto.PartitioningSubmitDTO import za.co.absa.atum.server.model.PartitioningForDB -import za.co.absa.fadb.DBSchema -import za.co.absa.fadb.doobie.{DoobieEngine, StatusWithData} -import za.co.absa.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus -import za.co.absa.fadb.status.handling.implementations.StandardStatusHandling +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.doobie.{DoobieEngine, StatusWithData} +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus +import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala index 29f0b450b..54c6671ac 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala @@ -24,9 +24,9 @@ import za.co.absa.atum.model.dto.PartitioningDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import za.co.absa.atum.server.model.PartitioningForDB -import za.co.absa.fadb.DBSchema -import za.co.absa.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction -import za.co.absa.fadb.doobie.DoobieEngine +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction +import za.co.absa.db.fadb.doobie.DoobieEngine import zio.interop.catz.asyncInstance import zio.{Task, URLayer, ZIO, ZLayer} import za.co.absa.atum.server.api.database.DoobieImplicits.getMapWithOptionStringValues diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 99ba17e59..de626d1c6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -24,9 +24,9 @@ import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import za.co.absa.atum.server.model.{CheckpointFromDB, PartitioningForDB} -import za.co.absa.fadb.DBSchema -import za.co.absa.fadb.doobie.DoobieEngine -import za.co.absa.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.doobie.DoobieEngine +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction import zio._ import zio.interop.catz._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala index 26274eac0..023a38c20 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala @@ -22,9 +22,9 @@ import doobie.util.Read import play.api.libs.json.Json import za.co.absa.atum.model.dto.{MeasureDTO, PartitioningDTO} import za.co.absa.atum.server.model.PartitioningForDB -import za.co.absa.fadb.DBSchema -import za.co.absa.fadb.doobie.DoobieEngine -import za.co.absa.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.doobie.DoobieEngine +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 35b615dad..45acefad1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -21,10 +21,10 @@ import doobie.implicits._ import doobie.util.Read import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.model.PartitioningForDB -import za.co.absa.fadb.DBSchema -import za.co.absa.fadb.doobie.{DoobieEngine, StatusWithData} -import za.co.absa.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus -import za.co.absa.fadb.status.handling.implementations.StandardStatusHandling +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.doobie.{DoobieEngine, StatusWithData} +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus +import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index 31116e7ea..c87000792 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.server.api.exception.DatabaseError -import za.co.absa.fadb.exceptions.StatusException +import za.co.absa.db.fadb.exceptions.StatusException import zio._ trait BaseRepository { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala index 26b551fcf..7d92ddf3f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.exception.DatabaseError -import za.co.absa.fadb.exceptions.StatusException +import za.co.absa.db.fadb.exceptions.StatusException import zio._ import zio.macros.accessible diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index e033f7f5f..e9f45f3f8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint import za.co.absa.atum.server.api.exception.DatabaseError -import za.co.absa.fadb.exceptions.StatusException +import za.co.absa.db.fadb.exceptions.StatusException import zio._ class CheckpointRepositoryImpl(writeCheckpointFn: WriteCheckpoint) extends CheckpointRepository with BaseRepository { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala index d61c13a6f..572cb9b5f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB -import za.co.absa.fadb.exceptions.StatusException +import za.co.absa.db.fadb.exceptions.StatusException import zio.IO import zio.macros.accessible diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index a9441d66b..765512fde 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -28,7 +28,7 @@ import za.co.absa.atum.server.api.database.runs.functions.{ GetPartitioningMeasures } import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB -import za.co.absa.fadb.exceptions.StatusException +import za.co.absa.db.fadb.exceptions.StatusException import zio._ import zio.prelude.ZivariantOps diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala index 3b070df36..66633cf78 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.server.api.exception.{DatabaseError, ServiceError} -import za.co.absa.fadb.exceptions.StatusException +import za.co.absa.db.fadb.exceptions.StatusException import zio._ trait BaseService { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala index 9d5bce9d1..cc90e943f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.exception.ServiceError -import za.co.absa.fadb.exceptions.StatusException +import za.co.absa.db.fadb.exceptions.StatusException import zio._ import zio.macros.accessible diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala index dba47bcd7..77701d350 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.repository.CheckpointRepository -import za.co.absa.fadb.exceptions.StatusException +import za.co.absa.db.fadb.exceptions.StatusException import zio._ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala index dd2e7c5c1..7ea6d295c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} import za.co.absa.atum.server.api.exception.ServiceError -import za.co.absa.fadb.exceptions.StatusException +import za.co.absa.db.fadb.exceptions.StatusException import zio.IO import zio.macros.accessible diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala index 687e1d629..0719bda04 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.repository.PartitioningRepository -import za.co.absa.fadb.exceptions.StatusException +import za.co.absa.db.fadb.exceptions.StatusException import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB import zio._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 80cdf125c..9f452e1da 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -21,8 +21,8 @@ import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.CheckpointService import za.co.absa.atum.server.model.ErrorResponse.{GeneralErrorResponse, InternalServerErrorResponse} -import za.co.absa.fadb.exceptions.ErrorInDataException -import za.co.absa.fadb.status.FunctionStatus +import za.co.absa.db.fadb.exceptions.ErrorInDataException +import za.co.absa.db.fadb.status.FunctionStatus import zio.test.Assertion.failsWithA import zio._ import zio.test._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalDataIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalDataIntegrationTests.scala index a0a282109..39559d72d 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalDataIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalDataIntegrationTests.scala @@ -16,12 +16,13 @@ package za.co.absa.atum.server.api.database.runs.functions +import doobie.WeakAsync.doobieWeakAsyncForAsync import za.co.absa.atum.model.dto.{AdditionalDataSubmitDTO, PartitionDTO} import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider -import za.co.absa.fadb.exceptions.DataNotFoundException -import za.co.absa.fadb.status.FunctionStatus +import za.co.absa.db.fadb.exceptions.DataNotFoundException +import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.test._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index f76e044e8..bf2d8386d 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -21,8 +21,8 @@ import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider -import za.co.absa.fadb.exceptions.DataNotFoundException -import za.co.absa.fadb.status.FunctionStatus +import za.co.absa.db.fadb.exceptions.DataNotFoundException +import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.test._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index fb4097e5f..2166169a0 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -20,8 +20,8 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData -import za.co.absa.fadb.exceptions.ErrorInDataException -import za.co.absa.fadb.status.FunctionStatus +import za.co.absa.db.fadb.exceptions.ErrorInDataException +import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.test.Assertion.failsWithA import zio.test._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala index aad01953a..568b7ed75 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala @@ -21,8 +21,8 @@ import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.database.runs.functions._ import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB -import za.co.absa.fadb.exceptions.ErrorInDataException -import za.co.absa.fadb.status.FunctionStatus +import za.co.absa.db.fadb.exceptions.ErrorInDataException +import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.test.Assertion.failsWithA import zio.test._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala index c68a2ede7..f8c420975 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala @@ -20,8 +20,8 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.{DatabaseError, ServiceError} import za.co.absa.atum.server.api.repository.CheckpointRepository -import za.co.absa.fadb.exceptions.ErrorInDataException -import za.co.absa.fadb.status.FunctionStatus +import za.co.absa.db.fadb.exceptions.ErrorInDataException +import za.co.absa.db.fadb.status.FunctionStatus import zio.test.Assertion.failsWithA import zio.test._ import zio._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala index 1308faaa6..f124a0ab5 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala @@ -20,8 +20,8 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.{DatabaseError, ServiceError} import za.co.absa.atum.server.api.repository.PartitioningRepository -import za.co.absa.fadb.exceptions.ErrorInDataException -import za.co.absa.fadb.status.FunctionStatus +import za.co.absa.db.fadb.exceptions.ErrorInDataException +import za.co.absa.db.fadb.status.FunctionStatus import zio.test.Assertion.failsWithA import zio.test._ import zio._ From a85e6728b4fad68837bd9a53ce45cb4069d73ee6 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 11 Jul 2024 22:38:29 +0200 Subject: [PATCH 02/31] Implicit encoders and decoders for PartitioningDTO --- model/src/main/scala/za/co/absa/atum/model/dto/package.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala index bca1a6c77..b89d71409 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala @@ -28,4 +28,7 @@ package object dto { implicit val decodeAdditionalDataDTO: Decoder[AdditionalDataDTO] = Decoder.decodeMap[String, Option[String]] implicit val encodeAdditionalDataDTO: Encoder[AdditionalDataDTO] = Encoder.encodeMap[String, Option[String]] + // Implicit encoders and decoders for PartitioningDTO + implicit val decodePartitioningDTO: Decoder[PartitioningDTO] = Decoder.decodeSeq[PartitionDTO] + implicit val encodePartitioningDTO: Encoder[PartitioningDTO] = Encoder.encodeSeq[PartitionDTO] } From 2de71dd2ed5d336de95005088ff76311c3af0439 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 12 Jul 2024 16:38:10 +0200 Subject: [PATCH 03/31] adding value parameter toFragmentsSeq --- .../za/co/absa/atum/server/api/database/DoobieImplicits.scala | 2 +- .../api/database/flows/functions/GetFlowCheckpoints.scala | 3 ++- .../database/runs/functions/CreateOrUpdateAdditionalData.scala | 2 +- .../runs/functions/CreatePartitioningIfNotExists.scala | 2 +- .../runs/functions/GetPartitioningAdditionalData.scala | 2 +- .../database/runs/functions/GetPartitioningCheckpoints.scala | 3 ++- .../api/database/runs/functions/GetPartitioningMeasures.scala | 2 +- .../server/api/database/runs/functions/WriteCheckpoint.scala | 2 +- 8 files changed, 10 insertions(+), 8 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala index 28163b264..2b896873b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala @@ -27,7 +27,7 @@ import io.circe.parser._ import scala.util.Try -package object DoobieImplicits { +object DoobieImplicits { private implicit val showPgArray: Show[PgArray] = Show.fromToString diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 49718bce4..0fac09e53 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -37,7 +37,8 @@ import doobie.postgres.circe.json.implicits.jsonGet import io.circe.syntax.EncoderOps class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunction[CheckpointQueryDTO, CheckpointFromDB, Task] { + extends DoobieMultipleResultFunction[CheckpointQueryDTO, CheckpointFromDB, Task](values => Seq(fr"$values")) + { override val fieldsToSelect: Seq[String] = Seq( "id_checkpoint", diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index c63717839..a94e9cac6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -35,7 +35,7 @@ import doobie.postgres.implicits._ import doobie.postgres.circe.jsonb.implicits.jsonbPut class CreateOrUpdateAdditionalData(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[AdditionalDataSubmitDTO, Unit, Task] + extends DoobieSingleResultFunctionWithStatus[AdditionalDataSubmitDTO, Unit, Task](values => Seq(fr"$values")) with StandardStatusHandling { override def sql(values: AdditionalDataSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 2bbcb59e3..feff2bccd 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -35,7 +35,7 @@ import io.circe.syntax._ import doobie.postgres.circe.jsonb.implicits.jsonbPut class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task] + extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task](values => Seq(fr"$values")) with StandardStatusHandling { override def sql(values: PartitioningSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala index a87b8c91e..1b5f0bbb0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala @@ -34,7 +34,7 @@ import za.co.absa.atum.server.api.database.DoobieImplicits.getMapWithOptionStrin import doobie.postgres.circe.jsonb.implicits.jsonbPut class GetPartitioningAdditionalData (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunction[PartitioningDTO, (String, Option[String]), Task] + extends DoobieMultipleResultFunction[PartitioningDTO, (String, Option[String]), Task](values => Seq(fr"$values")) { override val fieldsToSelect: Seq[String] = Seq("ad_name", "ad_value") diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 7c3587395..0409e80df 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -36,7 +36,8 @@ import doobie.postgres.circe.jsonb.implicits.jsonbPut import doobie.postgres.circe.json.implicits.jsonGet class GetPartitioningCheckpoints (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunction[CheckpointQueryDTO, CheckpointFromDB, Task] { + extends DoobieMultipleResultFunction[CheckpointQueryDTO, CheckpointFromDB, Task](values => Seq(fr"$values")) + { override val fieldsToSelect: Seq[String] = Seq( "id_checkpoint", diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala index 9c7c72029..1b0831398 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala @@ -34,7 +34,7 @@ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.circe.jsonb.implicits.jsonbPut class GetPartitioningMeasures (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunction[PartitioningDTO, MeasureDTO, Task] + extends DoobieMultipleResultFunction[PartitioningDTO, MeasureDTO, Task](values => Seq(fr"$values")) { override val fieldsToSelect: Seq[String] = Seq("measure_name", "measured_columns") diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index fbd829622..5d5f1db6c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -38,7 +38,7 @@ import doobie.postgres.circe.jsonb.implicits.jsonbPut import doobie.postgres.implicits._ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task] + extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task](values => Seq(fr"$values")) with StandardStatusHandling { override def sql(values: CheckpointDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { From 0176678168f3a3d90ed6a2210930d027b37057ff Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 15 Jul 2024 10:09:21 +0200 Subject: [PATCH 04/31] modifying the dbFunction call classes and PartitioningRepository --- .../flows/functions/GetFlowCheckpoints.scala | 8 ++--- .../GetPartitioningAdditionalData.scala | 7 ++-- .../GetPartitioningCheckpoints.scala | 7 ++-- .../functions/GetPartitioningMeasures.scala | 6 ++-- .../repository/PartitioningRepository.scala | 11 +++--- .../PartitioningRepositoryImpl.scala | 36 +++++++++---------- 6 files changed, 37 insertions(+), 38 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 0fac09e53..2391a335a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -25,22 +25,22 @@ import za.co.absa.atum.server.api.database.flows.Flows import za.co.absa.atum.server.model.{CheckpointFromDB, PartitioningForDB} import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine -import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWithAggStatus import zio._ import zio.interop.catz._ - import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get - import doobie.postgres.implicits._ import doobie.postgres.circe.jsonb.implicits.jsonbPut import doobie.postgres.circe.json.implicits.jsonGet import io.circe.syntax.EncoderOps class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunction[CheckpointQueryDTO, CheckpointFromDB, Task](values => Seq(fr"$values")) + extends DoobieMultipleResultFunctionWithAggStatus[CheckpointQueryDTO, CheckpointFromDB, Task](values => Seq(fr"$values")) { override val fieldsToSelect: Seq[String] = Seq( + "status", + "status_text", "id_checkpoint", "checkpoint_name", "author", diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala index 1b5f0bbb0..81a0b5dc8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala @@ -24,20 +24,19 @@ import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.db.fadb.DBSchema -import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWithAggStatus import za.co.absa.db.fadb.doobie.DoobieEngine import zio.interop.catz.asyncInstance import zio.{Task, URLayer, ZIO, ZLayer} import io.circe.syntax._ - import za.co.absa.atum.server.api.database.DoobieImplicits.getMapWithOptionStringValues import doobie.postgres.circe.jsonb.implicits.jsonbPut class GetPartitioningAdditionalData (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunction[PartitioningDTO, (String, Option[String]), Task](values => Seq(fr"$values")) + extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, (String, Option[String]), Task](values => Seq(fr"$values")) { - override val fieldsToSelect: Seq[String] = Seq("ad_name", "ad_value") + override val fieldsToSelect: Seq[String] = Seq("status", "status_text", "ad_name", "ad_value") override def sql(values: PartitioningDTO)(implicit read: Read[(String, Option[String])]): Fragment = { val partitioning: PartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(values) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 0409e80df..59cac3ad4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -25,21 +25,22 @@ import za.co.absa.atum.server.api.database.runs.Runs import za.co.absa.atum.server.model.{CheckpointFromDB, PartitioningForDB} import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine -import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWithAggStatus import zio._ import zio.interop.catz._ import io.circe.syntax._ - import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ import doobie.postgres.circe.jsonb.implicits.jsonbPut import doobie.postgres.circe.json.implicits.jsonGet class GetPartitioningCheckpoints (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunction[CheckpointQueryDTO, CheckpointFromDB, Task](values => Seq(fr"$values")) + extends DoobieMultipleResultFunctionWithAggStatus[CheckpointQueryDTO, CheckpointFromDB, Task](values => Seq(fr"$values")) { override val fieldsToSelect: Seq[String] = Seq( + "status", + "status_text", "id_checkpoint", "checkpoint_name", "author", diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala index 1b0831398..4f962f8ad 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala @@ -23,7 +23,7 @@ import za.co.absa.atum.model.dto.{MeasureDTO, PartitioningDTO} import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine -import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWithAggStatus import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ @@ -34,10 +34,10 @@ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.circe.jsonb.implicits.jsonbPut class GetPartitioningMeasures (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunction[PartitioningDTO, MeasureDTO, Task](values => Seq(fr"$values")) + extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, MeasureDTO, Task](values => Seq(fr"$values")) { - override val fieldsToSelect: Seq[String] = Seq("measure_name", "measured_columns") + override val fieldsToSelect: Seq[String] = Seq("status", "status_text", "measure_name", "measured_columns") override def sql(values: PartitioningDTO)(implicit read: Read[MeasureDTO]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala index 572cb9b5f..394d750f5 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala @@ -20,6 +20,7 @@ import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, Ch import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB import za.co.absa.db.fadb.exceptions.StatusException +import za.co.absa.db.fadb.status import zio.IO import zio.macros.accessible @@ -27,19 +28,19 @@ import zio.macros.accessible trait PartitioningRepository { def createPartitioningIfNotExists( partitioningSubmitDTO: PartitioningSubmitDTO - ): IO[DatabaseError, Either[StatusException, Unit]] + ): IO[DatabaseError, Either[StatusException, status.Row[Unit]]] def getPartitioningMeasures( partitioning: PartitioningDTO - ): IO[DatabaseError, Seq[MeasureDTO]] + ): IO[DatabaseError, Either[StatusException, Seq[status.Row[MeasureDTO]]]] def getPartitioningAdditionalData( partitioning: PartitioningDTO - ): IO[DatabaseError, AdditionalDataDTO] + ): IO[DatabaseError, Either[StatusException, status.Row[AdditionalDataDTO]]] def createOrUpdateAdditionalData(additionalData: AdditionalDataSubmitDTO): - IO[DatabaseError, Either[StatusException, Unit]] + IO[DatabaseError, Either[StatusException, status.Row[Unit]]] def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): - IO[DatabaseError, Seq[CheckpointFromDB]] + IO[DatabaseError, Either[StatusException, Seq[status.Row[CheckpointFromDB]]]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index 765512fde..a3d78f989 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -16,21 +16,17 @@ package za.co.absa.atum.server.api.repository -import za.co.absa.atum.model.dto.{ - AdditionalDataDTO, AdditionalDataSubmitDTO, - CheckpointQueryDTO, MeasureDTO, PartitioningDTO, - PartitioningSubmitDTO} -import za.co.absa.atum.server.api.database.runs.functions.{ - CreateOrUpdateAdditionalData, - CreatePartitioningIfNotExists, - GetPartitioningAdditionalData, - GetPartitioningCheckpoints, - GetPartitioningMeasures } +import shapeless.syntax.std.product.productOps +import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} +import za.co.absa.atum.server.api.database.runs.functions.{CreateOrUpdateAdditionalData, CreatePartitioningIfNotExists, GetPartitioningAdditionalData, GetPartitioningCheckpoints, GetPartitioningMeasures} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB import za.co.absa.db.fadb.exceptions.StatusException +import za.co.absa.db.fadb.status import zio._ +import zio.interop.catz.asyncInstance import zio.prelude.ZivariantOps +import zio.prelude.data.Optional.AllValuesAreNullable class PartitioningRepositoryImpl( createPartitioningIfNotExistsFn: CreatePartitioningIfNotExists, @@ -42,31 +38,33 @@ class PartitioningRepositoryImpl( override def createPartitioningIfNotExists( partitioningSubmitDTO: PartitioningSubmitDTO - ): IO[DatabaseError, Either[StatusException, Unit]] = { + ): IO[DatabaseError, Either[StatusException, status.Row[Unit]]] = { dbCallWithStatus(createPartitioningIfNotExistsFn(partitioningSubmitDTO), "createPartitioningIfNotExists") } override def createOrUpdateAdditionalData( additionalData: AdditionalDataSubmitDTO - ): IO[DatabaseError, Either[StatusException, Unit]] = { + ): IO[DatabaseError, Either[StatusException, status.Row[Unit]]] = { dbCallWithStatus(createOrUpdateAdditionalDataFn(additionalData), "createOrUpdateAdditionalData") } override def getPartitioningMeasures( partitioning: PartitioningDTO - ): IO[DatabaseError, Seq[MeasureDTO]] = { - val m = getPartitioningMeasuresFn(partitioning) - m.mapLeft(err => DatabaseError(err.getMessage)) + ): IO[DatabaseError, Either[StatusException, Seq[status.Row[MeasureDTO]]]] = { + dbCallWithStatus(getPartitioningMeasuresFn(partitioning), "getPartitioningMeasures") } override def getPartitioningAdditionalData(partitioning: PartitioningDTO): - IO[DatabaseError, AdditionalDataDTO] = { - getPartitioningAdditionalDataFn(partitioning).mapBoth(err => DatabaseError(err.getMessage), _.toMap) + IO[DatabaseError, Either[StatusException, status.Row[AdditionalDataDTO]]] = { + dbCallWithStatus(getPartitioningAdditionalDataFn(partitioning) + .map(_.toMap.getOrElse("additional_data", null)), + "getPartitioningAdditionalData") } + // mapBoth(err => DatabaseError(err.getMessage), _.toMap) override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): - IO[DatabaseError, Seq[CheckpointFromDB]] = { - dbCall(getPartitioningCheckpointsFn(checkpointQueryDTO), "getPartitioningCheckpoints") + IO[DatabaseError, Either[StatusException, Seq[status.Row[CheckpointFromDB]]]] = { + dbCallWithStatus(getPartitioningCheckpointsFn(checkpointQueryDTO), "getPartitioningCheckpoints") } } From 6a3474f135f1d86e95b23bef61e47a0a75555c4b Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 15 Jul 2024 10:28:30 +0200 Subject: [PATCH 05/31] Modifying getFlowCheckpoints --- .../server/api/repository/FlowRepository.scala | 5 ++++- .../api/repository/FlowRepositoryImpl.scala | 8 ++++++-- .../repository/PartitioningRepositoryImpl.scala | 17 +++++++++++++---- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepository.scala index 6f56aa145..5edf2506c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepository.scala @@ -19,10 +19,13 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB +import za.co.absa.db.fadb.exceptions.StatusException +import za.co.absa.db.fadb.status import zio._ import zio.macros.accessible @accessible trait FlowRepository { - def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] + def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): + IO[DatabaseError, Either[StatusException, Seq[status.Row[CheckpointFromDB]]]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala index 3a8e96a88..2d162b830 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala @@ -20,12 +20,16 @@ import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.database.flows.functions.GetFlowCheckpoints import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB +import za.co.absa.db.fadb.exceptions.StatusException +import za.co.absa.db.fadb.status import zio._ +import zio.interop.catz.asyncInstance class FlowRepositoryImpl(getFlowCheckpointsFn: GetFlowCheckpoints) extends FlowRepository with BaseRepository { - override def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] = { - dbCall(getFlowCheckpointsFn(checkpointQueryDTO), "getFlowCheckpoints") + override def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): + IO[DatabaseError, Either[StatusException, Seq[status.Row[CheckpointFromDB]]]] = { + dbCallWithStatus(getFlowCheckpointsFn(checkpointQueryDTO), "getFlowCheckpoints") } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index a3d78f989..60756d50e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -17,16 +17,25 @@ package za.co.absa.atum.server.api.repository import shapeless.syntax.std.product.productOps -import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} -import za.co.absa.atum.server.api.database.runs.functions.{CreateOrUpdateAdditionalData, CreatePartitioningIfNotExists, GetPartitioningAdditionalData, GetPartitioningCheckpoints, GetPartitioningMeasures} +import za.co.absa.atum.model.dto.{ + AdditionalDataDTO, + AdditionalDataSubmitDTO, + CheckpointQueryDTO, + MeasureDTO, + PartitioningDTO, + PartitioningSubmitDTO} +import za.co.absa.atum.server.api.database.runs.functions.{ + CreateOrUpdateAdditionalData, + CreatePartitioningIfNotExists, + GetPartitioningAdditionalData, + GetPartitioningCheckpoints, + GetPartitioningMeasures} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB import za.co.absa.db.fadb.exceptions.StatusException import za.co.absa.db.fadb.status import zio._ import zio.interop.catz.asyncInstance -import zio.prelude.ZivariantOps -import zio.prelude.data.Optional.AllValuesAreNullable class PartitioningRepositoryImpl( createPartitioningIfNotExistsFn: CreatePartitioningIfNotExists, From 1f6b20a3f79251325415c13e452091ed9abae89b Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 16 Jul 2024 14:29:01 +0200 Subject: [PATCH 06/31] Modifying partitioning service function signatures --- .../PartitioningControllerImpl.scala | 3 +- .../flows/functions/GetFlowCheckpoints.scala | 38 ++++---------- .../CreateOrUpdateAdditionalData.scala | 24 +++------ .../CreatePartitioningIfNotExists.scala | 26 +++------- .../GetPartitioningAdditionalData.scala | 34 +++++++------ .../GetPartitioningCheckpoints.scala | 50 +++++++++---------- .../functions/GetPartitioningMeasures.scala | 21 ++------ .../runs/functions/WriteCheckpoint.scala | 37 +++++--------- .../api/repository/CheckpointRepository.scala | 3 +- .../repository/CheckpointRepositoryImpl.scala | 4 +- .../PartitioningRepositoryImpl.scala | 1 - .../api/service/CheckpointService.scala | 3 +- .../api/service/CheckpointServiceImpl.scala | 3 +- .../server/api/service/FlowServiceImpl.scala | 2 +- .../api/service/PartitioningService.scala | 11 ++-- .../api/service/PartitioningServiceImpl.scala | 26 ++++++---- 16 files changed, 117 insertions(+), 169 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala index d37cc94a1..e45a1650d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala @@ -21,6 +21,7 @@ import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.PartitioningService import za.co.absa.atum.server.model.{ErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} +import za.co.absa.db.fadb.status import zio._ class PartitioningControllerImpl(partitioningService: PartitioningService) @@ -59,7 +60,7 @@ class PartitioningControllerImpl(partitioningService: PartitioningService) additionalData: AdditionalDataSubmitDTO ): IO[ErrorResponse, SingleSuccessResponse[AdditionalDataSubmitDTO]] = { mapToSingleSuccessResponse( - serviceCallWithStatus[Unit, AdditionalDataSubmitDTO]( + serviceCallWithStatus[status.Row[Unit], AdditionalDataSubmitDTO]( partitioningService.createOrUpdateAdditionalData(additionalData), _ => additionalData ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 2391a335a..2c479af69 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -16,9 +16,7 @@ package za.co.absa.atum.server.api.database.flows.functions -import doobie.Fragment import doobie.implicits.toSqlInterpolator -import doobie.util.Read import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.flows.Flows @@ -33,35 +31,17 @@ import doobie.postgres.implicits._ import doobie.postgres.circe.jsonb.implicits.jsonbPut import doobie.postgres.circe.json.implicits.jsonGet import io.circe.syntax.EncoderOps +import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusAggregator +import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunctionWithAggStatus[CheckpointQueryDTO, CheckpointFromDB, Task](values => Seq(fr"$values")) - { - - override val fieldsToSelect: Seq[String] = Seq( - "status", - "status_text", - "id_checkpoint", - "checkpoint_name", - "author", - "measured_by_atum_agent", - "measure_name", "measured_columns", "measurement_value", - "checkpoint_start_time", "checkpoint_end_time", - ) - - override def sql(values: CheckpointQueryDTO)(implicit read: Read[CheckpointFromDB]): Fragment = { - val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningNormalized = partitioning.asJson - - sql"""SELECT ${Fragment.const(selectEntry)} - FROM ${Fragment.const(functionName)}( - $partitioningNormalized, - ${values.limit}, - ${values.checkpointName} - ) AS ${Fragment.const(alias)};""" - } - -} + extends DoobieMultipleResultFunctionWithAggStatus[CheckpointQueryDTO, CheckpointFromDB, Task]( + values => Seq( + fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", + fr"${values.limit}", + fr"${values.checkpointName}" + ) + ) with StandardStatusHandling with ByFirstErrorStatusAggregator object GetFlowCheckpoints { val layer: URLayer[PostgresDatabaseProvider, GetFlowCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index a94e9cac6..793201241 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -35,23 +35,13 @@ import doobie.postgres.implicits._ import doobie.postgres.circe.jsonb.implicits.jsonbPut class CreateOrUpdateAdditionalData(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[AdditionalDataSubmitDTO, Unit, Task](values => Seq(fr"$values")) - with StandardStatusHandling { - - override def sql(values: AdditionalDataSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { - val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningJson = partitioning.asJson - - // implicits from Doobie can't handle Map[String, Option[String]] -> HStore, so we converted None to null basically - val additionalDataNormalized = values.additionalData.map{ case (k, v) => (k, v.orNull)} - - sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( - $partitioningJson, - $additionalDataNormalized, - ${values.author} - ) ${Fragment.const(alias)};""" - } -} + extends DoobieSingleResultFunctionWithStatus[AdditionalDataSubmitDTO, Unit, Task]( + values => Seq( + fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", + fr"${values.additionalData.map{ case (k, v) => (k, v.orNull)}}", + fr"${values.author}" + ) + ) with StandardStatusHandling object CreateOrUpdateAdditionalData { val layer: URLayer[PostgresDatabaseProvider, CreateOrUpdateAdditionalData] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index feff2bccd..fe8a478c3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -35,25 +35,13 @@ import io.circe.syntax._ import doobie.postgres.circe.jsonb.implicits.jsonbPut class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task](values => Seq(fr"$values")) - with StandardStatusHandling { - - override def sql(values: PartitioningSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { - val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningJson = partitioning.asJson - - val parentPartitioningJson = values.parentPartitioning.map { parentPartitioning => - val parentPartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(parentPartitioning) - parentPartitioningForDB.asJson - } - - sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( - $partitioningJson, - ${values.authorIfNew}, - $parentPartitioningJson - ) ${Fragment.const(alias)};""" - } -} + extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task]( + values => Seq( + fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", + fr"${values.authorIfNew}", + fr"${values.parentPartitioning.map(PartitioningForDB.fromSeqPartitionDTO).map(_.asJson)}" + ) + ) with StandardStatusHandling object CreatePartitioningIfNotExists { val layer: URLayer[PostgresDatabaseProvider, CreatePartitioningIfNotExists] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala index 81a0b5dc8..bbbc97665 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala @@ -31,23 +31,27 @@ import zio.{Task, URLayer, ZIO, ZLayer} import io.circe.syntax._ import za.co.absa.atum.server.api.database.DoobieImplicits.getMapWithOptionStringValues import doobie.postgres.circe.jsonb.implicits.jsonbPut +import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusAggregator +import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling class GetPartitioningAdditionalData (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, (String, Option[String]), Task](values => Seq(fr"$values")) - { - - override val fieldsToSelect: Seq[String] = Seq("status", "status_text", "ad_name", "ad_value") - - override def sql(values: PartitioningDTO)(implicit read: Read[(String, Option[String])]): Fragment = { - val partitioning: PartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(values) - val partitioningJson = partitioning.asJson - - sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( - $partitioningJson - ) ${Fragment.const(alias)};""" - } - -} + extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, (String, Option[String]), Task]( + values => Seq(fr"${PartitioningForDB.fromSeqPartitionDTO(values).asJson}")) + with StandardStatusHandling with ByFirstErrorStatusAggregator +// { +// +// override val fieldsToSelect: Seq[String] = Seq("status", "status_text", "ad_name", "ad_value") +// +// override def sql(values: PartitioningDTO)(implicit read: Read[(String, Option[String])]): Fragment = { +// val partitioning: PartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(values) +// val partitioningJson = partitioning.asJson +// +// sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( +// $partitioningJson +// ) ${Fragment.const(alias)};""" +// } +// +//} object GetPartitioningAdditionalData { val layer: URLayer[PostgresDatabaseProvider, GetPartitioningAdditionalData] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 59cac3ad4..7c8882881 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -23,7 +23,7 @@ import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import za.co.absa.atum.server.model.{CheckpointFromDB, PartitioningForDB} -import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.{DBSchema, status} import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWithAggStatus import zio._ @@ -33,37 +33,35 @@ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ import doobie.postgres.circe.jsonb.implicits.jsonbPut import doobie.postgres.circe.json.implicits.jsonGet +import za.co.absa.db.fadb.status.aggregation.implementations.{ByFirstErrorStatusAggregator, ByFirstRowStatusAggregator} +import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling +import za.co.absa.db.fadb.status.{FailedOrRow, FailedOrRows} class GetPartitioningCheckpoints (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunctionWithAggStatus[CheckpointQueryDTO, CheckpointFromDB, Task](values => Seq(fr"$values")) + extends DoobieMultipleResultFunctionWithAggStatus[CheckpointQueryDTO, CheckpointFromDB, Task] ( + values => Seq( + fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", + fr"${values.limit}", + fr"${values.checkpointName}" + ) + ) with StandardStatusHandling with ByFirstErrorStatusAggregator { - override val fieldsToSelect: Seq[String] = Seq( - "status", - "status_text", - "id_checkpoint", - "checkpoint_name", - "author", - "measured_by_atum_agent", - "measure_name", - "measured_columns", - "measurement_value", - "checkpoint_start_time", - "checkpoint_end_time", - ) +// override val fieldsToSelect: Seq[String] = Seq( +// "status", +// "status_text", +// "id_checkpoint", +// "checkpoint_name", +// "author", +// "measured_by_atum_agent", +// "measure_name", +// "measured_columns", +// "measurement_value", +// "checkpoint_start_time", +// "checkpoint_end_time", +// ) - override def sql(values: CheckpointQueryDTO)(implicit read: Read[CheckpointFromDB]): Fragment = { - val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningNormalized = partitioning.asJson - - sql"""SELECT ${Fragment.const(selectEntry)} - FROM ${Fragment.const(functionName)}( - $partitioningNormalized, - ${values.limit}, - ${values.checkpointName} - ) AS ${Fragment.const(alias)};""" } -} object GetPartitioningCheckpoints { val layer: URLayer[PostgresDatabaseProvider, GetPartitioningCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala index 4f962f8ad..9c7e1ac6b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala @@ -29,26 +29,15 @@ import za.co.absa.atum.server.api.database.runs.Runs import zio._ import zio.interop.catz._ import io.circe.syntax._ - import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.circe.jsonb.implicits.jsonbPut +import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusAggregator +import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling class GetPartitioningMeasures (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, MeasureDTO, Task](values => Seq(fr"$values")) - { - - override val fieldsToSelect: Seq[String] = Seq("status", "status_text", "measure_name", "measured_columns") - - override def sql(values: PartitioningDTO)(implicit read: Read[MeasureDTO]): Fragment = { - val partitioning = PartitioningForDB.fromSeqPartitionDTO(values) - val partitioningJson = partitioning.asJson - - sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( - $partitioningJson - ) ${Fragment.const(alias)};""" - } - -} + extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, MeasureDTO, Task]( + values => Seq(fr"${PartitioningForDB.fromSeqPartitionDTO(values).asJson}")) + with StandardStatusHandling with ByFirstErrorStatusAggregator object GetPartitioningMeasures { val layer: URLayer[PostgresDatabaseProvider, GetPartitioningMeasures] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 5d5f1db6c..62ff7c30a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -38,31 +38,18 @@ import doobie.postgres.circe.jsonb.implicits.jsonbPut import doobie.postgres.implicits._ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task](values => Seq(fr"$values")) - with StandardStatusHandling { - - override def sql(values: CheckpointDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { - val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningNormalized = partitioning.asJson - - // List[String] containing json data has to be properly escaped - // It would be safer to use Json data type and derive Put instance - val measurementsNormalized = { - values.measurements.toList.map(_.asJson) - } - - sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( - $partitioningNormalized, - ${values.id}, - ${values.name}, - ${values.processStartTime}, - ${values.processEndTime}, - $measurementsNormalized, - ${values.measuredByAtumAgent}, - ${values.author} - ) ${Fragment.const(alias)};""" - } -} + extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task]( + values => Seq( + fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", + fr"${values.id}", + fr"${values.name}", + fr"${values.processStartTime}", + fr"${values.processEndTime}", + fr"${values.measurements.toList.map(_.asJson)}", + fr"${values.measuredByAtumAgent}", + fr"${values.author}" + )) + with StandardStatusHandling object WriteCheckpoint { val layer: URLayer[PostgresDatabaseProvider, WriteCheckpoint] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala index 7d92ddf3f..120305e3a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala @@ -19,11 +19,12 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.db.fadb.exceptions.StatusException +import za.co.absa.db.fadb.status import zio._ import zio.macros.accessible @accessible trait CheckpointRepository { - def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Either[StatusException, Unit]] + def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Either[StatusException, status.Row[Unit]]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index e9f45f3f8..de91910c3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -20,11 +20,13 @@ import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.db.fadb.exceptions.StatusException +import za.co.absa.db.fadb.status import zio._ +import zio.interop.catz.asyncInstance class CheckpointRepositoryImpl(writeCheckpointFn: WriteCheckpoint) extends CheckpointRepository with BaseRepository { - override def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Either[StatusException, Unit]] = { + override def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Either[StatusException, status.Row[Unit]]] = { dbCallWithStatus(writeCheckpointFn(checkpointDTO), "writeCheckpoint") } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index 60756d50e..ecf37514e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -69,7 +69,6 @@ class PartitioningRepositoryImpl( .map(_.toMap.getOrElse("additional_data", null)), "getPartitioningAdditionalData") } - // mapBoth(err => DatabaseError(err.getMessage), _.toMap) override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Either[StatusException, Seq[status.Row[CheckpointFromDB]]]] = { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala index cc90e943f..857f8b778 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala @@ -19,11 +19,12 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.db.fadb.exceptions.StatusException +import za.co.absa.db.fadb.status import zio._ import zio.macros.accessible @accessible trait CheckpointService { - def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Either[StatusException, Unit]] + def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Either[StatusException, status.Row[Unit]]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala index 77701d350..fb2f55d97 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala @@ -20,12 +20,13 @@ import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.repository.CheckpointRepository import za.co.absa.db.fadb.exceptions.StatusException +import za.co.absa.db.fadb.status import zio._ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends CheckpointService with BaseService { - override def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Either[StatusException, Unit]] = { + override def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Either[StatusException, status.Row[Unit]]] = { repositoryCallWithStatus( checkpointRepository.writeCheckpoint(checkpointDTO), "saveCheckpoint" ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala index 93e333c67..6459b8848 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala @@ -32,7 +32,7 @@ class FlowServiceImpl(flowRepository: FlowRepository) flowRepository.getFlowCheckpoints(checkpointQueryDTO), "getFlowCheckpoints" ) checkpointDTOs <- ZIO.foreach(checkpointsFromDB) { - checkpointFromDB => + checkpointFromDB: => ZIO.fromEither(CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, checkpointFromDB)) .mapError(error => ServiceError(error.getMessage)) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala index 7ea6d295c..3cda6ef14 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala @@ -19,22 +19,23 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.db.fadb.exceptions.StatusException -import zio.IO +import za.co.absa.db.fadb.status +import zio.{IO, ZIO} import zio.macros.accessible @accessible trait PartitioningService { def createPartitioningIfNotExists(partitioningSubmitDTO: PartitioningSubmitDTO): - IO[ServiceError, Either[StatusException, Unit]] + IO[ServiceError, Either[StatusException, status.Row[Unit]]] def getPartitioningMeasures(partitioning: PartitioningDTO): - IO[ServiceError, Seq[MeasureDTO]] + IO[ServiceError, Either[StatusException, Seq[status.Row[MeasureDTO]]]] def getPartitioningAdditionalData(partitioning: PartitioningDTO): - IO[ServiceError, AdditionalDataDTO] + IO[ServiceError, Either[StatusException, status.Row[AdditionalDataDTO]]] def createOrUpdateAdditionalData(additionalData: AdditionalDataSubmitDTO): - IO[ServiceError, Either[StatusException, Unit]] + IO[ServiceError, Either[StatusException, status.Row[Unit]]] def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[ServiceError, Seq[CheckpointDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala index 0719bda04..8731c1802 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala @@ -22,13 +22,14 @@ import za.co.absa.atum.server.api.repository.PartitioningRepository import za.co.absa.db.fadb.exceptions.StatusException import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB +import za.co.absa.db.fadb.status import zio._ class PartitioningServiceImpl(partitioningRepository: PartitioningRepository) extends PartitioningService with BaseService { override def createPartitioningIfNotExists(partitioningSubmitDTO: PartitioningSubmitDTO): - IO[ServiceError, Either[StatusException, Unit]] = { + IO[ServiceError, Either[StatusException, status.Row[Unit]]] = { repositoryCallWithStatus( partitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO), "createPartitioningIfNotExists" ).mapError(error => ServiceError(error.message)) @@ -36,20 +37,25 @@ class PartitioningServiceImpl(partitioningRepository: PartitioningRepository) override def createOrUpdateAdditionalData( additionalData: AdditionalDataSubmitDTO - ): IO[ServiceError, Either[StatusException, Unit]] = { + ): IO[ServiceError, Either[StatusException, status.Row[Unit]]] = { repositoryCallWithStatus( partitioningRepository.createOrUpdateAdditionalData(additionalData), "createOrUpdateAdditionalData" ).mapError(error => ServiceError(error.message)) } - override def getPartitioningMeasures(partitioning: PartitioningDTO): IO[ServiceError, Seq[MeasureDTO]] = { - partitioningRepository.getPartitioningMeasures(partitioning) - .mapError { case DatabaseError(message) => - ServiceError(s"Failed to retrieve partitioning measures': $message") - } + override def getPartitioningMeasures(partitioning: PartitioningDTO): + IO[ServiceError, Either[StatusException, Seq[status.Row[MeasureDTO]]]] = { +// partitioningRepository.getPartitioningMeasures(partitioning) +// .mapError { case DatabaseError(message) => +// ServiceError(s"Failed to retrieve partitioning measures': $message") +// } + repositoryCallWithStatus( + partitioningRepository.getPartitioningMeasures(partitioning), "getPartitioningMeasures" + ).mapError (error => ServiceError(error.message)) } - override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[ServiceError, AdditionalDataDTO] = { + override def getPartitioningAdditionalData(partitioning: PartitioningDTO): + IO[ServiceError, Either[StatusException, status.Row[AdditionalDataDTO]]] = { partitioningRepository.getPartitioningAdditionalData(partitioning) .mapError { case DatabaseError(message) => ServiceError(s"Failed to retrieve partitioning additional data': $message") @@ -58,9 +64,9 @@ class PartitioningServiceImpl(partitioningRepository: PartitioningRepository) override def getPartitioningCheckpoints( checkpointQueryDTO: CheckpointQueryDTO - ): IO[ServiceError, Seq[CheckpointDTO]] = { + ): IO[ServiceError, Either[StatusException, Seq[status.Row[CheckpointDTO]]]] = { for { - checkpointsFromDB <- repositoryCall( + checkpointsFromDB <- repositoryCallWithStatus( partitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO), "getPartitioningCheckpoints" ) checkpointDTOs <- ZIO.foreach(checkpointsFromDB) { From b1bbd321f6935c8457fb9f2a5940fda4a7af50b6 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 17 Jul 2024 15:14:41 +0200 Subject: [PATCH 07/31] partitioning service --- .../functions/GetPartitioningCheckpoints.scala | 17 ----------------- .../atum/server/api/service/FlowService.scala | 6 ++++-- .../server/api/service/FlowServiceImpl.scala | 10 +++++++--- .../api/service/PartitioningServiceImpl.scala | 15 ++++++--------- .../atum/server/model/CheckpointFromDB.scala | 1 + 5 files changed, 18 insertions(+), 31 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 7c8882881..1539b599b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -45,23 +45,6 @@ class GetPartitioningCheckpoints (implicit schema: DBSchema, dbEngine: DoobieEng fr"${values.checkpointName}" ) ) with StandardStatusHandling with ByFirstErrorStatusAggregator - { - -// override val fieldsToSelect: Seq[String] = Seq( -// "status", -// "status_text", -// "id_checkpoint", -// "checkpoint_name", -// "author", -// "measured_by_atum_agent", -// "measure_name", -// "measured_columns", -// "measurement_value", -// "checkpoint_start_time", -// "checkpoint_end_time", -// ) - - } object GetPartitioningCheckpoints { val layer: URLayer[PostgresDatabaseProvider, GetPartitioningCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowService.scala index 34781cbdf..1f385b6d5 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowService.scala @@ -16,12 +16,14 @@ package za.co.absa.atum.server.api.service -import za.co.absa.atum.model.dto.{CheckpointQueryDTO, CheckpointDTO} +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointQueryDTO} import za.co.absa.atum.server.api.exception.ServiceError +import za.co.absa.db.fadb.exceptions.StatusException +import za.co.absa.db.fadb.status import zio._ import zio.macros.accessible @accessible trait FlowService { - def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[ServiceError, Seq[CheckpointDTO]] + def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[ServiceError, Either[StatusException, Seq[status.Row[CheckpointDTO]]]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala index 6459b8848..307eb29d5 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala @@ -20,19 +20,23 @@ import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.repository.FlowRepository import za.co.absa.atum.server.model.CheckpointFromDB +import za.co.absa.db.fadb.exceptions.StatusException +import za.co.absa.db.fadb.status import zio._ class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with BaseService { - override def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[ServiceError, Seq[CheckpointDTO]] = { + override def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): + IO[ServiceError, Either[StatusException, Seq[status.Row[CheckpointDTO]]]] = { for { checkpointsFromDB <- repositoryCall( flowRepository.getFlowCheckpoints(checkpointQueryDTO), "getFlowCheckpoints" - ) + ).mapError(error => ServiceError(error.message)) + .flatMap(ZIO.fromEither(_)) checkpointDTOs <- ZIO.foreach(checkpointsFromDB) { - checkpointFromDB: => + checkpointFromDB => ZIO.fromEither(CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, checkpointFromDB)) .mapError(error => ServiceError(error.getMessage)) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala index 8731c1802..463d385f0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala @@ -65,16 +65,13 @@ class PartitioningServiceImpl(partitioningRepository: PartitioningRepository) override def getPartitioningCheckpoints( checkpointQueryDTO: CheckpointQueryDTO ): IO[ServiceError, Either[StatusException, Seq[status.Row[CheckpointDTO]]]] = { - for { - checkpointsFromDB <- repositoryCallWithStatus( + repositoryCallWithStatus( partitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO), "getPartitioningCheckpoints" - ) - checkpointDTOs <- ZIO.foreach(checkpointsFromDB) { - checkpointFromDB => - ZIO.fromEither(CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, checkpointFromDB)) - .mapError(error => ServiceError(error.getMessage)) - } - } yield checkpointDTOs + ).mapBoth(error => ServiceError(error.getMessage), { + case Left(value) => Left(value) + case Right(value) => Right(value.map(x => CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, x.data))) + }) + } } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 705e6c319..7752335f7 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -21,6 +21,7 @@ import io.circe.{DecodingFailure, Json} import java.time.ZonedDateTime import java.util.UUID +import scala.Right case class CheckpointFromDB( idCheckpoint: UUID, From 166f50661abcb87a81365ec7c98429ee1bd59eb9 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 18 Jul 2024 10:28:00 +0200 Subject: [PATCH 08/31] Refactoring --- .../api/repository/BaseRepository.scala | 24 ++++++++++++++++++- .../repository/PartitioningRepository.scala | 18 +++++++------- .../PartitioningRepositoryImpl.scala | 14 +++++------ .../api/service/PartitioningServiceImpl.scala | 6 ++--- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index c87000792..2aed1dcbc 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -18,6 +18,7 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.db.fadb.exceptions.StatusException +import zio.ZIO.{fail, succeed} import zio._ trait BaseRepository { @@ -32,7 +33,7 @@ trait BaseRepository { .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } - def dbCallWithStatus[R]( + def dbCallWithStatusR[R]( dbFuncCall: Task[Either[StatusException, R]], operationName: String ): IO[DatabaseError, Either[StatusException, R]] = { @@ -48,4 +49,25 @@ trait BaseRepository { .mapError(error => DatabaseError(error.getMessage)) .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } + + def dbCallWithStatus[R]( + dbFuncCall: Task[Either[StatusException, R]], + operationName: String + ): IO[DatabaseError, R] = { + dbFuncCall + .tap { + case Left(statusException) => + ZIO.logError( + s"Exception caused by operation: '$operationName': " + + s"(${statusException.status.statusCode}) ${statusException.status.statusText}" + ) + case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") + } + .mapError(error => DatabaseError(error.getMessage)) + .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) + .flatMap { + case Left(_) => fail(DatabaseError("Failed to execute operation in database")) + case Right(value) => succeed(value) + } + } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala index 394d750f5..49bd7cd66 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala @@ -19,8 +19,6 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB -import za.co.absa.db.fadb.exceptions.StatusException -import za.co.absa.db.fadb.status import zio.IO import zio.macros.accessible @@ -28,19 +26,21 @@ import zio.macros.accessible trait PartitioningRepository { def createPartitioningIfNotExists( partitioningSubmitDTO: PartitioningSubmitDTO - ): IO[DatabaseError, Either[StatusException, status.Row[Unit]]] + ): IO[DatabaseError, Unit] def getPartitioningMeasures( partitioning: PartitioningDTO - ): IO[DatabaseError, Either[StatusException, Seq[status.Row[MeasureDTO]]]] + ): IO[DatabaseError, Seq[MeasureDTO]] def getPartitioningAdditionalData( partitioning: PartitioningDTO - ): IO[DatabaseError, Either[StatusException, status.Row[AdditionalDataDTO]]] + ): IO[DatabaseError, AdditionalDataDTO] - def createOrUpdateAdditionalData(additionalData: AdditionalDataSubmitDTO): - IO[DatabaseError, Either[StatusException, status.Row[Unit]]] + def createOrUpdateAdditionalData( + additionalData: AdditionalDataSubmitDTO + ): IO[DatabaseError, Unit] - def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): - IO[DatabaseError, Either[StatusException, Seq[status.Row[CheckpointFromDB]]]] + def getPartitioningCheckpoints( + checkpointQueryDTO: CheckpointQueryDTO + ): IO[DatabaseError, Seq[CheckpointFromDB]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index ecf37514e..daec41a60 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -47,31 +47,31 @@ class PartitioningRepositoryImpl( override def createPartitioningIfNotExists( partitioningSubmitDTO: PartitioningSubmitDTO - ): IO[DatabaseError, Either[StatusException, status.Row[Unit]]] = { + ): IO[DatabaseError, Unit] = { dbCallWithStatus(createPartitioningIfNotExistsFn(partitioningSubmitDTO), "createPartitioningIfNotExists") } override def createOrUpdateAdditionalData( additionalData: AdditionalDataSubmitDTO - ): IO[DatabaseError, Either[StatusException, status.Row[Unit]]] = { + ): IO[DatabaseError, Unit] = { dbCallWithStatus(createOrUpdateAdditionalDataFn(additionalData), "createOrUpdateAdditionalData") } override def getPartitioningMeasures( partitioning: PartitioningDTO - ): IO[DatabaseError, Either[StatusException, Seq[status.Row[MeasureDTO]]]] = { + ): IO[DatabaseError, Seq[MeasureDTO]] = { dbCallWithStatus(getPartitioningMeasuresFn(partitioning), "getPartitioningMeasures") } override def getPartitioningAdditionalData(partitioning: PartitioningDTO): - IO[DatabaseError, Either[StatusException, status.Row[AdditionalDataDTO]]] = { + IO[DatabaseError, AdditionalDataDTO] = { dbCallWithStatus(getPartitioningAdditionalDataFn(partitioning) - .map(_.toMap.getOrElse("additional_data", null)), - "getPartitioningAdditionalData") + .map(_.toMap.getOrElse("additional_data", null)), "getPartitioningAdditionalData" + ) } override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): - IO[DatabaseError, Either[StatusException, Seq[status.Row[CheckpointFromDB]]]] = { + IO[DatabaseError, Seq[CheckpointFromDB]] = { dbCallWithStatus(getPartitioningCheckpointsFn(checkpointQueryDTO), "getPartitioningCheckpoints") } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala index 463d385f0..98cdd491a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala @@ -69,10 +69,10 @@ class PartitioningServiceImpl(partitioningRepository: PartitioningRepository) partitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO), "getPartitioningCheckpoints" ).mapBoth(error => ServiceError(error.getMessage), { case Left(value) => Left(value) - case Right(value) => Right(value.map(x => CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, x.data))) + case Right(value) => Right(value.map(x => CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, x.data) + .flatMap(_) + )) }) - - } } From 953a99a46e7779645c85d19a55cdbb3bc09fe8d0 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 18 Jul 2024 15:14:44 +0200 Subject: [PATCH 09/31] Refactoring repository services --- .../api/repository/BaseRepository.scala | 17 ------ .../api/repository/FlowRepository.scala | 4 +- .../api/repository/FlowRepositoryImpl.scala | 5 +- .../PartitioningRepositoryImpl.scala | 24 +++------ .../atum/server/api/service/FlowService.scala | 6 +-- .../server/api/service/FlowServiceImpl.scala | 10 ++-- .../api/service/PartitioningService.scala | 12 ++--- .../api/service/PartitioningServiceImpl.scala | 53 +++++++++---------- .../atum/server/model/CheckpointFromDB.scala | 1 - 9 files changed, 45 insertions(+), 87 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index 2aed1dcbc..0b5a20c3e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -33,23 +33,6 @@ trait BaseRepository { .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } - def dbCallWithStatusR[R]( - dbFuncCall: Task[Either[StatusException, R]], - operationName: String - ): IO[DatabaseError, Either[StatusException, R]] = { - dbFuncCall - .tap { - case Left(statusException) => - ZIO.logError( - s"Exception caused by operation: '$operationName': " + - s"(${statusException.status.statusCode}) ${statusException.status.statusText}" - ) - case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") - } - .mapError(error => DatabaseError(error.getMessage)) - .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) - } - def dbCallWithStatus[R]( dbFuncCall: Task[Either[StatusException, R]], operationName: String diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepository.scala index 5edf2506c..f00e644c5 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepository.scala @@ -19,13 +19,11 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB -import za.co.absa.db.fadb.exceptions.StatusException -import za.co.absa.db.fadb.status import zio._ import zio.macros.accessible @accessible trait FlowRepository { def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): - IO[DatabaseError, Either[StatusException, Seq[status.Row[CheckpointFromDB]]]] + IO[DatabaseError, Seq[CheckpointFromDB]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala index 2d162b830..e18f06ade 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala @@ -20,16 +20,15 @@ import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.database.flows.functions.GetFlowCheckpoints import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB -import za.co.absa.db.fadb.exceptions.StatusException -import za.co.absa.db.fadb.status import zio._ import zio.interop.catz.asyncInstance class FlowRepositoryImpl(getFlowCheckpointsFn: GetFlowCheckpoints) extends FlowRepository with BaseRepository { override def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): - IO[DatabaseError, Either[StatusException, Seq[status.Row[CheckpointFromDB]]]] = { + IO[DatabaseError, Seq[CheckpointFromDB]] = { dbCallWithStatus(getFlowCheckpointsFn(checkpointQueryDTO), "getFlowCheckpoints") + .map(_.map(x => x.data)) } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index daec41a60..7d227bed1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -17,23 +17,10 @@ package za.co.absa.atum.server.api.repository import shapeless.syntax.std.product.productOps -import za.co.absa.atum.model.dto.{ - AdditionalDataDTO, - AdditionalDataSubmitDTO, - CheckpointQueryDTO, - MeasureDTO, - PartitioningDTO, - PartitioningSubmitDTO} -import za.co.absa.atum.server.api.database.runs.functions.{ - CreateOrUpdateAdditionalData, - CreatePartitioningIfNotExists, - GetPartitioningAdditionalData, - GetPartitioningCheckpoints, - GetPartitioningMeasures} +import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} +import za.co.absa.atum.server.api.database.runs.functions.{CreateOrUpdateAdditionalData, CreatePartitioningIfNotExists, GetPartitioningAdditionalData, GetPartitioningCheckpoints, GetPartitioningMeasures} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB -import za.co.absa.db.fadb.exceptions.StatusException -import za.co.absa.db.fadb.status import zio._ import zio.interop.catz.asyncInstance @@ -48,19 +35,20 @@ class PartitioningRepositoryImpl( override def createPartitioningIfNotExists( partitioningSubmitDTO: PartitioningSubmitDTO ): IO[DatabaseError, Unit] = { - dbCallWithStatus(createPartitioningIfNotExistsFn(partitioningSubmitDTO), "createPartitioningIfNotExists") + dbCallWithStatus(createPartitioningIfNotExistsFn(partitioningSubmitDTO).map(_.map(x => x.data)), "createPartitioningIfNotExists") } override def createOrUpdateAdditionalData( additionalData: AdditionalDataSubmitDTO ): IO[DatabaseError, Unit] = { - dbCallWithStatus(createOrUpdateAdditionalDataFn(additionalData), "createOrUpdateAdditionalData") + dbCallWithStatus(createOrUpdateAdditionalDataFn(additionalData).map(_.map(x => x.data)), "createOrUpdateAdditionalData") } override def getPartitioningMeasures( partitioning: PartitioningDTO ): IO[DatabaseError, Seq[MeasureDTO]] = { dbCallWithStatus(getPartitioningMeasuresFn(partitioning), "getPartitioningMeasures") + .map(_.map(x => x.data)) } override def getPartitioningAdditionalData(partitioning: PartitioningDTO): @@ -73,7 +61,9 @@ class PartitioningRepositoryImpl( override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] = { dbCallWithStatus(getPartitioningCheckpointsFn(checkpointQueryDTO), "getPartitioningCheckpoints") + .map(_.map(x => x.data)) } + } object PartitioningRepositoryImpl { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowService.scala index 1f385b6d5..1464aed78 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowService.scala @@ -18,12 +18,12 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointQueryDTO} import za.co.absa.atum.server.api.exception.ServiceError -import za.co.absa.db.fadb.exceptions.StatusException -import za.co.absa.db.fadb.status import zio._ import zio.macros.accessible @accessible trait FlowService { - def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[ServiceError, Either[StatusException, Seq[status.Row[CheckpointDTO]]]] + def getFlowCheckpoints( + checkpointQueryDTO: CheckpointQueryDTO + ): IO[ServiceError, Seq[CheckpointDTO]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala index 307eb29d5..93e333c67 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala @@ -20,23 +20,19 @@ import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.repository.FlowRepository import za.co.absa.atum.server.model.CheckpointFromDB -import za.co.absa.db.fadb.exceptions.StatusException -import za.co.absa.db.fadb.status import zio._ class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with BaseService { - override def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): - IO[ServiceError, Either[StatusException, Seq[status.Row[CheckpointDTO]]]] = { + override def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[ServiceError, Seq[CheckpointDTO]] = { for { checkpointsFromDB <- repositoryCall( flowRepository.getFlowCheckpoints(checkpointQueryDTO), "getFlowCheckpoints" - ).mapError(error => ServiceError(error.message)) - .flatMap(ZIO.fromEither(_)) + ) checkpointDTOs <- ZIO.foreach(checkpointsFromDB) { - checkpointFromDB => + checkpointFromDB => ZIO.fromEither(CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, checkpointFromDB)) .mapError(error => ServiceError(error.getMessage)) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala index 3cda6ef14..4a87ecd9d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala @@ -18,24 +18,22 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} import za.co.absa.atum.server.api.exception.ServiceError -import za.co.absa.db.fadb.exceptions.StatusException -import za.co.absa.db.fadb.status -import zio.{IO, ZIO} +import zio.IO import zio.macros.accessible @accessible trait PartitioningService { def createPartitioningIfNotExists(partitioningSubmitDTO: PartitioningSubmitDTO): - IO[ServiceError, Either[StatusException, status.Row[Unit]]] + IO[ServiceError, Unit] def getPartitioningMeasures(partitioning: PartitioningDTO): - IO[ServiceError, Either[StatusException, Seq[status.Row[MeasureDTO]]]] + IO[ServiceError, Seq[MeasureDTO]] def getPartitioningAdditionalData(partitioning: PartitioningDTO): - IO[ServiceError, Either[StatusException, status.Row[AdditionalDataDTO]]] + IO[ServiceError, AdditionalDataDTO] def createOrUpdateAdditionalData(additionalData: AdditionalDataSubmitDTO): - IO[ServiceError, Either[StatusException, status.Row[Unit]]] + IO[ServiceError, Unit] def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[ServiceError, Seq[CheckpointDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala index 98cdd491a..b1470fee9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala @@ -19,60 +19,55 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.repository.PartitioningRepository -import za.co.absa.db.fadb.exceptions.StatusException -import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB -import za.co.absa.db.fadb.status import zio._ class PartitioningServiceImpl(partitioningRepository: PartitioningRepository) extends PartitioningService with BaseService { override def createPartitioningIfNotExists(partitioningSubmitDTO: PartitioningSubmitDTO): - IO[ServiceError, Either[StatusException, status.Row[Unit]]] = { - repositoryCallWithStatus( + IO[ServiceError, Unit] = { + repositoryCall( partitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO), "createPartitioningIfNotExists" - ).mapError(error => ServiceError(error.message)) + ) } override def createOrUpdateAdditionalData( additionalData: AdditionalDataSubmitDTO - ): IO[ServiceError, Either[StatusException, status.Row[Unit]]] = { - repositoryCallWithStatus( + ): IO[ServiceError, Unit] = { + repositoryCall( partitioningRepository.createOrUpdateAdditionalData(additionalData), "createOrUpdateAdditionalData" - ).mapError(error => ServiceError(error.message)) + ) } override def getPartitioningMeasures(partitioning: PartitioningDTO): - IO[ServiceError, Either[StatusException, Seq[status.Row[MeasureDTO]]]] = { -// partitioningRepository.getPartitioningMeasures(partitioning) -// .mapError { case DatabaseError(message) => -// ServiceError(s"Failed to retrieve partitioning measures': $message") -// } - repositoryCallWithStatus( + IO[ServiceError, Seq[MeasureDTO]] = { + repositoryCall( partitioningRepository.getPartitioningMeasures(partitioning), "getPartitioningMeasures" - ).mapError (error => ServiceError(error.message)) + ) } override def getPartitioningAdditionalData(partitioning: PartitioningDTO): - IO[ServiceError, Either[StatusException, status.Row[AdditionalDataDTO]]] = { - partitioningRepository.getPartitioningAdditionalData(partitioning) - .mapError { case DatabaseError(message) => - ServiceError(s"Failed to retrieve partitioning additional data': $message") - } + IO[ServiceError, AdditionalDataDTO] = { + repositoryCall( + partitioningRepository.getPartitioningAdditionalData(partitioning), "getPartitioningAdditionalData" + ) } override def getPartitioningCheckpoints( checkpointQueryDTO: CheckpointQueryDTO - ): IO[ServiceError, Either[StatusException, Seq[status.Row[CheckpointDTO]]]] = { - repositoryCallWithStatus( + ): IO[ServiceError, Seq[CheckpointDTO]] = { + for { + checkpointsFromDB <- repositoryCall( partitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO), "getPartitioningCheckpoints" - ).mapBoth(error => ServiceError(error.getMessage), { - case Left(value) => Left(value) - case Right(value) => Right(value.map(x => CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, x.data) - .flatMap(_) - )) - }) + ) + checkpointDTOs <- ZIO.foreach(checkpointsFromDB) { + checkpointFromDB => + ZIO.fromEither(CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, checkpointFromDB)) + .mapError(error => ServiceError(error.getMessage)) + } + } yield checkpointDTOs + } } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 7752335f7..705e6c319 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -21,7 +21,6 @@ import io.circe.{DecodingFailure, Json} import java.time.ZonedDateTime import java.util.UUID -import scala.Right case class CheckpointFromDB( idCheckpoint: UUID, From 075c62117c4d0dfc5d086d76204abd86ef201810 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 18 Jul 2024 16:15:14 +0200 Subject: [PATCH 10/31] refactoring serviceCallWithStatus in controller --- .../server/api/controller/CheckpointControllerImpl.scala | 2 +- .../server/api/controller/PartitioningControllerImpl.scala | 3 +-- .../atum/server/api/repository/CheckpointRepository.scala | 4 +--- .../server/api/repository/CheckpointRepositoryImpl.scala | 6 ++---- .../co/absa/atum/server/api/service/CheckpointService.scala | 6 ++---- .../atum/server/api/service/CheckpointServiceImpl.scala | 6 ++---- 6 files changed, 9 insertions(+), 18 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala index 8815a89c2..b64c825a7 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala @@ -27,7 +27,7 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che override def createCheckpointV1( checkpointDTO: CheckpointDTO ): IO[ErrorResponse, CheckpointDTO] = { - serviceCallWithStatus[Unit, CheckpointDTO]( + serviceCall[Unit, CheckpointDTO]( checkpointService.saveCheckpoint(checkpointDTO), _ => checkpointDTO ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala index e45a1650d..6fe9efff9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala @@ -21,7 +21,6 @@ import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.PartitioningService import za.co.absa.atum.server.model.{ErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} -import za.co.absa.db.fadb.status import zio._ class PartitioningControllerImpl(partitioningService: PartitioningService) @@ -60,7 +59,7 @@ class PartitioningControllerImpl(partitioningService: PartitioningService) additionalData: AdditionalDataSubmitDTO ): IO[ErrorResponse, SingleSuccessResponse[AdditionalDataSubmitDTO]] = { mapToSingleSuccessResponse( - serviceCallWithStatus[status.Row[Unit], AdditionalDataSubmitDTO]( + serviceCall[Unit, AdditionalDataSubmitDTO]( partitioningService.createOrUpdateAdditionalData(additionalData), _ => additionalData ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala index 120305e3a..59c33d1b6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala @@ -18,13 +18,11 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.exception.DatabaseError -import za.co.absa.db.fadb.exceptions.StatusException -import za.co.absa.db.fadb.status import zio._ import zio.macros.accessible @accessible trait CheckpointRepository { - def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Either[StatusException, status.Row[Unit]]] + def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index de91910c3..0cc22e5e0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -19,15 +19,13 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint import za.co.absa.atum.server.api.exception.DatabaseError -import za.co.absa.db.fadb.exceptions.StatusException -import za.co.absa.db.fadb.status import zio._ import zio.interop.catz.asyncInstance class CheckpointRepositoryImpl(writeCheckpointFn: WriteCheckpoint) extends CheckpointRepository with BaseRepository { - override def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Either[StatusException, status.Row[Unit]]] = { - dbCallWithStatus(writeCheckpointFn(checkpointDTO), "writeCheckpoint") + override def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] = { + dbCallWithStatus(writeCheckpointFn(checkpointDTO).map(_.map(x => x.data)), "writeCheckpoint") } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala index 857f8b778..a38811890 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala @@ -18,13 +18,11 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.exception.ServiceError -import za.co.absa.db.fadb.exceptions.StatusException -import za.co.absa.db.fadb.status -import zio._ +import zio.IO import zio.macros.accessible @accessible trait CheckpointService { - def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Either[StatusException, status.Row[Unit]]] + def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala index fb2f55d97..0616e8adf 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala @@ -19,15 +19,13 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.repository.CheckpointRepository -import za.co.absa.db.fadb.exceptions.StatusException -import za.co.absa.db.fadb.status import zio._ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends CheckpointService with BaseService { - override def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Either[StatusException, status.Row[Unit]]] = { - repositoryCallWithStatus( + override def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] = { + repositoryCall( checkpointRepository.writeCheckpoint(checkpointDTO), "saveCheckpoint" ) } From 933688bb556115fc6fe869f6680d648a6f88151b Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 19 Jul 2024 12:47:01 +0200 Subject: [PATCH 11/31] incorporating statusCode dbCallWithStatus --- .../GetPartitioningCheckpoints.scala | 5 ++-- .../api/repository/BaseRepository.scala | 18 ++++++++++-- .../PartitioningRepositoryImpl.scala | 28 ++++++++++++++++--- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 1539b599b..701ef73ec 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -23,7 +23,7 @@ import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import za.co.absa.atum.server.model.{CheckpointFromDB, PartitioningForDB} -import za.co.absa.db.fadb.{DBSchema, status} +import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWithAggStatus import zio._ @@ -33,9 +33,8 @@ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ import doobie.postgres.circe.jsonb.implicits.jsonbPut import doobie.postgres.circe.json.implicits.jsonGet -import za.co.absa.db.fadb.status.aggregation.implementations.{ByFirstErrorStatusAggregator, ByFirstRowStatusAggregator} +import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling -import za.co.absa.db.fadb.status.{FailedOrRow, FailedOrRows} class GetPartitioningCheckpoints (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunctionWithAggStatus[CheckpointQueryDTO, CheckpointFromDB, Task] ( diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index 0b5a20c3e..57448fcf1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -42,15 +42,27 @@ trait BaseRepository { case Left(statusException) => ZIO.logError( s"Exception caused by operation: '$operationName': " + - s"(${statusException.status.statusCode}) ${statusException.status.statusText}" + s"(${statusException.status.statusCode}), ${statusException.status.statusText}" ) case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") } - .mapError(error => DatabaseError(error.getMessage)) + .mapError { + case statusException: StatusException => + DatabaseError( + s"Exception caused by operation: '$operationName': " + + s"(${statusException.status.statusCode}) ${statusException.status.statusText}" + ) + case error => + DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") + } .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) .flatMap { - case Left(_) => fail(DatabaseError("Failed to execute operation in database")) + case Left(statusException) => fail(DatabaseError( + s"Failed to execute operation in database: " + + s"(${statusException.status.statusCode}) ${statusException.status.statusText}" + )) case Right(value) => succeed(value) } } + } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index 7d227bed1..8bf051c19 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -17,8 +17,19 @@ package za.co.absa.atum.server.api.repository import shapeless.syntax.std.product.productOps -import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} -import za.co.absa.atum.server.api.database.runs.functions.{CreateOrUpdateAdditionalData, CreatePartitioningIfNotExists, GetPartitioningAdditionalData, GetPartitioningCheckpoints, GetPartitioningMeasures} +import za.co.absa.atum.model.dto.{ + AdditionalDataDTO, + AdditionalDataSubmitDTO, + CheckpointQueryDTO, + MeasureDTO, + PartitioningDTO, + PartitioningSubmitDTO} +import za.co.absa.atum.server.api.database.runs.functions.{ + CreateOrUpdateAdditionalData, + CreatePartitioningIfNotExists, + GetPartitioningAdditionalData, + GetPartitioningCheckpoints, + GetPartitioningMeasures} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB import zio._ @@ -54,9 +65,18 @@ class PartitioningRepositoryImpl( override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[DatabaseError, AdditionalDataDTO] = { dbCallWithStatus(getPartitioningAdditionalDataFn(partitioning) - .map(_.toMap.getOrElse("additional_data", null)), "getPartitioningAdditionalData" - ) + .map(_.toMap.getOrElse("additional_data", null)), "getPartitioningAdditionalData") } +// override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[DatabaseError, AdditionalDataDTO] = { +// dbCallWithStatus(getPartitioningAdditionalDataFn(partitioning) +// .map { data => +// implicitly[ToMap[FailedOrRows[(String, Option[String])]]] +// .apply(data) +// .get(Symbol("additional_data")) +// .getOrElse(null) +// }, "getPartitioningAdditionalData" +// ) +// } override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] = { From d70a8c1abe3335b90c240adeb8ad0e6913d8ae78 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 23 Jul 2024 15:17:47 +0200 Subject: [PATCH 12/31] refactoring partitioning and test cases --- .../api/repository/BaseRepository.scala | 48 +++++++++++++++---- .../PartitioningRepositoryImpl.scala | 47 ++++++++---------- .../CheckpointControllerUnitTests.scala | 11 ++--- .../PartitioningServiceUnitTests.scala | 20 ++++---- 4 files changed, 72 insertions(+), 54 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index 57448fcf1..f6d857692 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -18,7 +18,6 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.db.fadb.exceptions.StatusException -import zio.ZIO.{fail, succeed} import zio._ trait BaseRepository { @@ -33,6 +32,38 @@ trait BaseRepository { .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } +// def dbCallWithStatus[R]( +// dbFuncCall: Task[Either[StatusException, R]], +// operationName: String +// ): IO[DatabaseError, R] = { +// dbFuncCall +// .tap { +// case Left(statusException) => +// ZIO.logError( +// s"Exception caused by operation: '$operationName': " + +// s"(${statusException.status.statusCode}), ${statusException.status.statusText}" +// ) +// case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") +// } +// .mapError { +// case statusException: StatusException => +// DatabaseError( +// s"Exception caused by operation: '$operationName': " + +// s"(${statusException.status.statusCode}) ${statusException.status.statusText}" +// ) +// case error => +// DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") +// } +// .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) +// .flatMap { +// case Left(statusException) => fail(DatabaseError( +// s"Failed to execute operation in database: " + +// s"(${statusException.status.statusCode}) ${statusException.status.statusText}" +// )) +// case Right(value) => succeed(value) +// } +// } + def dbCallWithStatus[R]( dbFuncCall: Task[Either[StatusException, R]], operationName: String @@ -46,6 +77,14 @@ trait BaseRepository { ) case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") } + .flatMap { + case Left(statusException) => + ZIO.fail(DatabaseError( + s"Failed to execute operation in database: " + + s"(${statusException.status.statusCode}) ${statusException.status.statusText}" + )) + case Right(value) => ZIO.succeed(value) + } .mapError { case statusException: StatusException => DatabaseError( @@ -56,13 +95,6 @@ trait BaseRepository { DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") } .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) - .flatMap { - case Left(statusException) => fail(DatabaseError( - s"Failed to execute operation in database: " + - s"(${statusException.status.statusCode}) ${statusException.status.statusText}" - )) - case Right(value) => succeed(value) - } } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index 8bf051c19..7d0be1ce2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -16,25 +16,14 @@ package za.co.absa.atum.server.api.repository -import shapeless.syntax.std.product.productOps -import za.co.absa.atum.model.dto.{ - AdditionalDataDTO, - AdditionalDataSubmitDTO, - CheckpointQueryDTO, - MeasureDTO, - PartitioningDTO, - PartitioningSubmitDTO} -import za.co.absa.atum.server.api.database.runs.functions.{ - CreateOrUpdateAdditionalData, - CreatePartitioningIfNotExists, - GetPartitioningAdditionalData, - GetPartitioningCheckpoints, - GetPartitioningMeasures} +import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} +import za.co.absa.atum.server.api.database.runs.functions.{CreateOrUpdateAdditionalData, CreatePartitioningIfNotExists, GetPartitioningAdditionalData, GetPartitioningCheckpoints, GetPartitioningMeasures} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB import zio._ import zio.interop.catz.asyncInstance + class PartitioningRepositoryImpl( createPartitioningIfNotExistsFn: CreatePartitioningIfNotExists, getPartitioningMeasuresFn: GetPartitioningMeasures, @@ -62,28 +51,30 @@ class PartitioningRepositoryImpl( .map(_.map(x => x.data)) } - override def getPartitioningAdditionalData(partitioning: PartitioningDTO): - IO[DatabaseError, AdditionalDataDTO] = { - dbCallWithStatus(getPartitioningAdditionalDataFn(partitioning) - .map(_.toMap.getOrElse("additional_data", null)), "getPartitioningAdditionalData") - } -// override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[DatabaseError, AdditionalDataDTO] = { -// dbCallWithStatus(getPartitioningAdditionalDataFn(partitioning) -// .map { data => -// implicitly[ToMap[FailedOrRows[(String, Option[String])]]] -// .apply(data) -// .get(Symbol("additional_data")) -// .getOrElse(null) -// }, "getPartitioningAdditionalData" -// ) +// override def getPartitioningAdditionalData(partitioning: PartitioningDTO): +// IO[DatabaseError, AdditionalDataDTO] = { +// getPartitioningAdditionalDataFn(partitioning) +// .flatMap { +// case Left(value) => ZIO.fail(DatabaseError(s"Failed to retrieve data with status code ${value.status}")) +// case Right(value) => ZIO.succeed(value.map(_.data).toMap) +// }.mapError(error => DatabaseError(error.getMessage)) // } + override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[DatabaseError, AdditionalDataDTO] = { + dbCallWithStatus( + getPartitioningAdditionalDataFn(partitioning) + .map(_.map(_.map(_.data).toMap)), "getPartitioningAdditionalData" + ) + } + + override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] = { dbCallWithStatus(getPartitioningCheckpointsFn(checkpointQueryDTO), "getPartitioningCheckpoints") .map(_.map(x => x.data)) } + } object PartitioningRepositoryImpl { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 9f452e1da..9f4a023e2 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -20,9 +20,6 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.CheckpointService -import za.co.absa.atum.server.model.ErrorResponse.{GeneralErrorResponse, InternalServerErrorResponse} -import za.co.absa.db.fadb.exceptions.ErrorInDataException -import za.co.absa.db.fadb.status.FunctionStatus import zio.test.Assertion.failsWithA import zio._ import zio.test._ @@ -31,9 +28,9 @@ object CheckpointControllerUnitTests extends ZIOSpecDefault with TestData { private val checkpointServiceMock = mock(classOf[CheckpointService]) - when(checkpointServiceMock.saveCheckpoint(checkpointDTO1)).thenReturn(ZIO.right(())) + when(checkpointServiceMock.saveCheckpoint(checkpointDTO1)).thenReturn(ZIO.succeed(())) when(checkpointServiceMock.saveCheckpoint(checkpointDTO2)) - .thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in data")))) + .thenReturn(ZIO.fail(ServiceError("error in data"))) when(checkpointServiceMock.saveCheckpoint(checkpointDTO3)) .thenReturn(ZIO.fail(ServiceError("boom!"))) @@ -49,10 +46,10 @@ object CheckpointControllerUnitTests extends ZIOSpecDefault with TestData { } yield assertTrue(result == checkpointDTO1) }, test("Returns expected InternalServerErrorResponse") { - assertZIO(CheckpointController.createCheckpointV1(checkpointDTO3).exit)(failsWithA[InternalServerErrorResponse]) + assertZIO(CheckpointController.createCheckpointV1(checkpointDTO3).exit)(failsWithA[ServiceError]) }, test("Returns expected GeneralErrorResponse") { - assertZIO(CheckpointController.createCheckpointV1(checkpointDTO2).exit)(failsWithA[GeneralErrorResponse]) + assertZIO(CheckpointController.createCheckpointV1(checkpointDTO2).exit)(failsWithA[ServiceError]) } ) ).provide( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala index f124a0ab5..7f8682387 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala @@ -20,9 +20,7 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.{DatabaseError, ServiceError} import za.co.absa.atum.server.api.repository.PartitioningRepository -import za.co.absa.db.fadb.exceptions.ErrorInDataException -import za.co.absa.db.fadb.status.FunctionStatus -import zio.test.Assertion.failsWithA +import zio.test.Assertion.{equalTo, failsWithA, isLeft, isRight, isUnit} import zio.test._ import zio._ @@ -30,15 +28,15 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { private val partitioningRepositoryMock = mock(classOf[PartitioningRepository]) - when(partitioningRepositoryMock.createPartitioningIfNotExists(partitioningSubmitDTO1)).thenReturn(ZIO.right(())) + when(partitioningRepositoryMock.createPartitioningIfNotExists(partitioningSubmitDTO1)).thenReturn(ZIO.succeed(())) when(partitioningRepositoryMock.createPartitioningIfNotExists(partitioningSubmitDTO2)) - .thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in data")))) + .thenReturn(ZIO.fail(DatabaseError("error in data"))) when(partitioningRepositoryMock.createPartitioningIfNotExists(partitioningSubmitDTO3)) .thenReturn(ZIO.fail(DatabaseError("boom!"))) - when(partitioningRepositoryMock.createOrUpdateAdditionalData(additionalDataSubmitDTO1)).thenReturn(ZIO.right(())) + when(partitioningRepositoryMock.createOrUpdateAdditionalData(additionalDataSubmitDTO1)).thenReturn(ZIO.succeed(())) when(partitioningRepositoryMock.createOrUpdateAdditionalData(additionalDataSubmitDTO2)) - .thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in AD data")))) + .thenReturn(ZIO.fail(DatabaseError("error in AD data"))) when(partitioningRepositoryMock.createOrUpdateAdditionalData(additionalDataSubmitDTO3)) .thenReturn(ZIO.fail(DatabaseError("boom!"))) @@ -66,12 +64,12 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO1) - } yield assertTrue(result.isRight) + } yield assert(result)(isRight(isUnit)) }, test("Returns expected Left with StatusException") { for { result <- PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO2) - } yield assertTrue(result.isLeft) + } yield assert(result)(isLeft(equalTo(DatabaseError("error in data")))) }, test("Returns expected ServiceError") { assertZIO(PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO3).exit)( @@ -83,12 +81,12 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO1) - } yield assertTrue(result.isRight) + } yield assert(result)(isRight(isUnit)) }, test("Returns expected Left with StatusException") { for { result <- PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO2) - } yield assertTrue(result.isLeft) + } yield assert(result)(isLeft(equalTo(DatabaseError("error in AD data")))) }, test("Returns expected ServiceError") { assertZIO(PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO3).exit)( From c5dc61fcd35c57d0a9724e88707fbe40c2e0579f Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 24 Jul 2024 10:21:08 +0200 Subject: [PATCH 13/31] Fixing test cases --- .../PartitioningControllerUnitTests.scala | 4 ++-- ...UpdateAdditionalDataIntegrationTests.scala | 2 +- ...titioningIfNotExistsIntegrationTests.scala | 1 + ...ioningAdditionalDataIntegrationTests.scala | 1 + ...titioningCheckpointsIntegrationTests.scala | 1 + ...PartitioningMeasuresIntegrationTests.scala | 1 + .../WriteCheckpointIntegrationTests.scala | 1 + .../CheckpointRepositoryUnitTests.scala | 15 ++++++------ .../repository/FlowRepositoryUnitTests.scala | 5 +++- .../PartitioningRepositoryUnitTests.scala | 24 +++++++++---------- .../service/CheckpointServiceUnitTests.scala | 14 +++++------ .../PartitioningServiceUnitTests.scala | 8 +++---- 12 files changed, 41 insertions(+), 36 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala index 7de77eb40..333c79ea6 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala @@ -31,7 +31,7 @@ object PartitioningControllerUnitTests extends ZIOSpecDefault with TestData { private val partitioningServiceMock = mock(classOf[PartitioningService]) when(partitioningServiceMock.createPartitioningIfNotExists(partitioningSubmitDTO1)) - .thenReturn(ZIO.right(())) + .thenReturn(ZIO.succeed(())) when(partitioningServiceMock.createPartitioningIfNotExists(partitioningSubmitDTO2)) .thenReturn(ZIO.fail(ServiceError("boom!"))) @@ -42,7 +42,7 @@ object PartitioningControllerUnitTests extends ZIOSpecDefault with TestData { .thenReturn(ZIO.succeed(Map.empty)) when(partitioningServiceMock.createOrUpdateAdditionalData(additionalDataSubmitDTO1)) - .thenReturn(ZIO.right(())) + .thenReturn(ZIO.succeed(())) when(partitioningServiceMock.createOrUpdateAdditionalData(additionalDataSubmitDTO2)) .thenReturn(ZIO.fail(ServiceError("boom!"))) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalDataIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalDataIntegrationTests.scala index 39559d72d..344b6048b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalDataIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalDataIntegrationTests.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.server.api.database.runs.functions -import doobie.WeakAsync.doobieWeakAsyncForAsync import za.co.absa.atum.model.dto.{AdditionalDataSubmitDTO, PartitionDTO} import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider @@ -24,6 +23,7 @@ import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.db.fadb.exceptions.DataNotFoundException import za.co.absa.db.fadb.status.FunctionStatus import zio._ +import zio.interop.catz.asyncInstance import zio.test._ object CreateOrUpdateAdditionalDataIntegrationTests extends ConfigProviderTest { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExistsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExistsIntegrationTests.scala index fa3f9dddb..32361e75f 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExistsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExistsIntegrationTests.scala @@ -21,6 +21,7 @@ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import zio._ +import zio.interop.catz.asyncInstance import zio.test._ object CreatePartitioningIfNotExistsIntegrationTests extends ConfigProviderTest { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalDataIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalDataIntegrationTests.scala index 045f74c69..133324989 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalDataIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalDataIntegrationTests.scala @@ -21,6 +21,7 @@ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import zio._ +import zio.interop.catz.asyncInstance import zio.test._ import zio.test.Assertion._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala index 911581c9b..b7da127af 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala @@ -20,6 +20,7 @@ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.model.dto.{CheckpointQueryDTO, PartitionDTO, PartitioningDTO} import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import zio.interop.catz.asyncInstance import zio.test.Assertion.failsWithA import zio.{Scope, ZIO} import zio.test._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala index 35d4c4c8a..aedc0e249 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala @@ -20,6 +20,7 @@ import za.co.absa.atum.model.dto.{PartitionDTO, PartitioningDTO} import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import zio.interop.catz.asyncInstance import zio.test.Assertion.failsWithA import zio.test.{Spec, TestEnvironment, assert} import zio.{Scope, ZIO} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index 6c397dd9f..c2e423f37 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -25,6 +25,7 @@ import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.db.fadb.exceptions.DataNotFoundException import za.co.absa.db.fadb.status.FunctionStatus import zio._ +import zio.interop.catz.asyncInstance import zio.test._ import java.time.ZonedDateTime diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index 2166169a0..9000e4348 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -23,18 +23,17 @@ import za.co.absa.atum.server.api.TestData import za.co.absa.db.fadb.exceptions.ErrorInDataException import za.co.absa.db.fadb.status.FunctionStatus import zio._ -import zio.test.Assertion.failsWithA +import zio.interop.catz.asyncInstance +import zio.test.Assertion.{failsWithA, isUnit} import zio.test._ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { private val writeCheckpointMock: WriteCheckpoint = mock(classOf[WriteCheckpoint]) - when(writeCheckpointMock.apply(checkpointDTO1)).thenReturn(ZIO.right(())) - when(writeCheckpointMock.apply(checkpointDTO2)) - .thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in data")))) - when(writeCheckpointMock.apply(checkpointDTO3)) - .thenReturn(ZIO.fail(new Exception("boom!"))) + when(writeCheckpointMock.apply(checkpointDTO1)).thenAnswer(_ => ZIO.succeed(())) + when(writeCheckpointMock.apply(checkpointDTO2)).thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in data")))) + when(writeCheckpointMock.apply(checkpointDTO3)).thenReturn(ZIO.fail(new Exception("boom!"))) private val writeCheckpointMockLayer = ZLayer.succeed(writeCheckpointMock) @@ -45,12 +44,12 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- CheckpointRepository.writeCheckpoint(checkpointDTO1) - } yield assertTrue(result.isRight) + } yield assert(result)(isUnit) }, test("Returns expected Left with StatusException") { for { result <- CheckpointRepository.writeCheckpoint(checkpointDTO2) - } yield assertTrue(result.isLeft) + } yield assert(result)(isUnit) }, test("Returns expected DatabaseError") { assertZIO(CheckpointRepository.writeCheckpoint(checkpointDTO3).exit)(failsWithA[DatabaseError]) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/FlowRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/FlowRepositoryUnitTests.scala index a0baa3be7..d1945d424 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/FlowRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/FlowRepositoryUnitTests.scala @@ -17,10 +17,13 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} +import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.database.flows.functions.GetFlowCheckpoints import za.co.absa.atum.server.api.exception.DatabaseError +import za.co.absa.atum.server.model.CheckpointFromDB import zio._ +import zio.interop.catz.asyncInstance import zio.test.Assertion.failsWithA import zio.test._ @@ -29,7 +32,7 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { private val getFlowCheckpointsMock = mock(classOf[GetFlowCheckpoints]) when(getFlowCheckpointsMock.apply(checkpointQueryDTO1)).thenReturn(ZIO.fail(new Exception("boom!"))) - when(getFlowCheckpointsMock.apply(checkpointQueryDTO2)).thenReturn(ZIO.succeed(Seq(checkpointFromDB1, checkpointFromDB2))) + when(getFlowCheckpointsMock.apply(checkpointQueryDTO2)).thenAnswer(_ => ZIO.succeed(Seq(checkpointFromDB1, checkpointFromDB2))) private val getFlowCheckpointsMockLayer = ZLayer.succeed(getFlowCheckpointsMock) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala index 568b7ed75..2b352d303 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala @@ -24,7 +24,8 @@ import za.co.absa.atum.server.model.CheckpointFromDB import za.co.absa.db.fadb.exceptions.ErrorInDataException import za.co.absa.db.fadb.status.FunctionStatus import zio._ -import zio.test.Assertion.failsWithA +import zio.interop.catz.asyncInstance +import zio.test.Assertion.{failsWithA, isRight, isUnit} import zio.test._ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { @@ -32,7 +33,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { // Create Partitioning Mocks private val createPartitioningIfNotExistsMock = mock(classOf[CreatePartitioningIfNotExists]) - when(createPartitioningIfNotExistsMock.apply(partitioningSubmitDTO1)).thenReturn(ZIO.right(())) + when(createPartitioningIfNotExistsMock.apply(partitioningSubmitDTO1)).thenAnswer(_ => ZIO.right(())) when(createPartitioningIfNotExistsMock.apply(partitioningSubmitDTO2)) .thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in Partitioning data")))) when(createPartitioningIfNotExistsMock.apply(partitioningSubmitDTO3)) @@ -43,7 +44,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { // Create Additional Data Mocks private val createOrUpdateAdditionalDataMock = mock(classOf[CreateOrUpdateAdditionalData]) - when(createOrUpdateAdditionalDataMock.apply(additionalDataSubmitDTO1)).thenReturn(ZIO.right(())) + when(createOrUpdateAdditionalDataMock.apply(additionalDataSubmitDTO1)).thenAnswer(_ => ZIO.right(())) when(createOrUpdateAdditionalDataMock.apply(additionalDataSubmitDTO2)) .thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in AD data")))) when(createOrUpdateAdditionalDataMock.apply(additionalDataSubmitDTO3)) @@ -54,7 +55,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { // Get Partitioning Measures Mocks private val getPartitioningMeasuresMock = mock(classOf[GetPartitioningMeasures]) - when(getPartitioningMeasuresMock.apply(partitioningDTO1)).thenReturn(ZIO.succeed(Seq(measureDTO1, measureDTO2))) + when(getPartitioningMeasuresMock.apply(partitioningDTO1)).thenAnswer(_ => ZIO.right(Seq(measureDTO1, measureDTO2))) when(getPartitioningMeasuresMock.apply(partitioningDTO2)).thenReturn(ZIO.fail(DatabaseError("boom!"))) private val getPartitioningMeasuresMockLayer = ZLayer.succeed(getPartitioningMeasuresMock) @@ -62,8 +63,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { // Get Partitioning Additional Data Mocks private val getPartitioningAdditionalDataMock = mock(classOf[GetPartitioningAdditionalData]) - when(getPartitioningAdditionalDataMock.apply(partitioningDTO1)) - .thenReturn(ZIO.succeed(additionalDataDTOSeq1)) + when(getPartitioningAdditionalDataMock.apply(partitioningDTO1)).thenAnswer(_ => ZIO.right(additionalDataDTOSeq1)) when(getPartitioningAdditionalDataMock.apply(partitioningDTO2)).thenReturn(ZIO.fail(DatabaseError("boom!"))) private val getPartitioningAdditionalDataMockLayer = ZLayer.succeed(getPartitioningAdditionalDataMock) @@ -71,9 +71,9 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { // Get Partitioning Checkpoints Mocks private val getPartitioningCheckpointsMock = mock(classOf[GetPartitioningCheckpoints]) - when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO1)).thenReturn(ZIO.succeed(Seq(checkpointFromDB1))) + when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO1)).thenAnswer(_ => ZIO.succeed(Seq(checkpointFromDB1))) when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO2)).thenReturn(ZIO.fail(DatabaseError("boom!"))) - when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO3)).thenReturn(ZIO.succeed(Seq.empty)) + when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO3)).thenAnswer(_ => ZIO.succeed(Seq.empty)) private val getPartitioningCheckpointsMockLayer = ZLayer.succeed(getPartitioningCheckpointsMock) @@ -84,12 +84,12 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- PartitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO1) - } yield assertTrue(result.isRight) + } yield assert(result)(isUnit) }, test("Returns expected Left with StatusException") { for { result <- PartitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO2) - } yield assertTrue(result.isLeft) + } yield assert(result)(isUnit) }, test("Returns expected DatabaseError") { assertZIO(PartitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO3).exit)( @@ -102,12 +102,12 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- PartitioningRepository.createOrUpdateAdditionalData(additionalDataSubmitDTO1) - } yield assertTrue(result.isRight) + } yield assert(result)(isUnit) }, test("Returns expected Left with StatusException") { for { result <- PartitioningRepository.createOrUpdateAdditionalData(additionalDataSubmitDTO2) - } yield assertTrue(result.isLeft) + } yield assert(result)(isUnit) }, test("Returns expected DatabaseError") { assertZIO(PartitioningRepository.createOrUpdateAdditionalData(additionalDataSubmitDTO3).exit)( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala index f8c420975..7d730b655 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala @@ -22,7 +22,7 @@ import za.co.absa.atum.server.api.exception.{DatabaseError, ServiceError} import za.co.absa.atum.server.api.repository.CheckpointRepository import za.co.absa.db.fadb.exceptions.ErrorInDataException import za.co.absa.db.fadb.status.FunctionStatus -import zio.test.Assertion.failsWithA +import zio.test.Assertion.{failsWithA, isLeft, isRight, isUnit} import zio.test._ import zio._ @@ -30,11 +30,9 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { private val checkpointRepositoryMock = mock(classOf[CheckpointRepository]) - when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO1)).thenReturn(ZIO.right(())) - when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO2)) - .thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in data")))) - when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO3)) - .thenReturn(ZIO.fail(DatabaseError("boom!"))) + when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO1)).thenAnswer(_ => ZIO.right(())) + when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO2)).thenReturn(ZIO.fail(DatabaseError("error in data"))) + when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO3)).thenReturn(ZIO.fail(DatabaseError("boom!"))) private val checkpointRepositoryMockLayer = ZLayer.succeed(checkpointRepositoryMock) @@ -45,12 +43,12 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- CheckpointService.saveCheckpoint(checkpointDTO1) - } yield assertTrue(result.isRight) + } yield assert(result)(isUnit) }, test("Returns expected Left with StatusException") { for { result <- CheckpointService.saveCheckpoint(checkpointDTO2) - } yield assertTrue(result.isLeft) + } yield assert(result)(isUnit) }, test("Returns expected ServiceError") { assertZIO(CheckpointService.saveCheckpoint(checkpointDTO3).exit)(failsWithA[ServiceError]) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala index 7f8682387..341a476b5 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala @@ -64,12 +64,12 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO1) - } yield assert(result)(isRight(isUnit)) + } yield assert(result)(isUnit) }, test("Returns expected Left with StatusException") { for { result <- PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO2) - } yield assert(result)(isLeft(equalTo(DatabaseError("error in data")))) + } yield assert(result)(isUnit) }, test("Returns expected ServiceError") { assertZIO(PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO3).exit)( @@ -81,12 +81,12 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO1) - } yield assert(result)(isRight(isUnit)) + } yield assert(result)(isUnit) }, test("Returns expected Left with StatusException") { for { result <- PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO2) - } yield assert(result)(isLeft(equalTo(DatabaseError("error in AD data")))) + } yield assert(result)(isUnit) }, test("Returns expected ServiceError") { assertZIO(PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO3).exit)( From b31d9a587ad3158998523a883d894bfa9f7fb7de Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 26 Jul 2024 08:03:50 +0200 Subject: [PATCH 14/31] Fixing PartitioningRepositoryUnitTests --- .../GetPartitioningAdditionalData.scala | 14 -- .../runs/functions/WriteCheckpoint.scala | 3 +- .../api/repository/BaseRepository.scala | 130 +++++++++++++----- .../repository/CheckpointRepositoryImpl.scala | 2 +- .../api/repository/FlowRepositoryImpl.scala | 3 +- .../PartitioningRepositoryImpl.scala | 26 ++-- .../CheckpointControllerUnitTests.scala | 5 +- .../CheckpointRepositoryUnitTests.scala | 7 +- .../repository/FlowRepositoryUnitTests.scala | 9 +- .../PartitioningRepositoryUnitTests.scala | 54 +++++--- 10 files changed, 150 insertions(+), 103 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala index bbbc97665..ae06790bd 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala @@ -38,20 +38,6 @@ class GetPartitioningAdditionalData (implicit schema: DBSchema, dbEngine: Doobie extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, (String, Option[String]), Task]( values => Seq(fr"${PartitioningForDB.fromSeqPartitionDTO(values).asJson}")) with StandardStatusHandling with ByFirstErrorStatusAggregator -// { -// -// override val fieldsToSelect: Seq[String] = Seq("status", "status_text", "ad_name", "ad_value") -// -// override def sql(values: PartitioningDTO)(implicit read: Read[(String, Option[String])]): Fragment = { -// val partitioning: PartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(values) -// val partitioningJson = partitioning.asJson -// -// sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( -// $partitioningJson -// ) ${Fragment.const(alias)};""" -// } -// -//} object GetPartitioningAdditionalData { val layer: URLayer[PostgresDatabaseProvider, GetPartitioningAdditionalData] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 62ff7c30a..3f64e6284 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -18,11 +18,10 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.Fragment import doobie.implicits.toSqlInterpolator -import doobie.util.Read import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.db.fadb.DBSchema -import za.co.absa.db.fadb.doobie.{DoobieEngine, StatusWithData} +import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import za.co.absa.atum.server.api.database.PostgresDatabaseProvider diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index f6d857692..8990a6f74 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -18,11 +18,23 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.db.fadb.exceptions.StatusException +import za.co.absa.db.fadb.status.{FailedOrRow, FailedOrRows} import zio._ +// R - dbSingleResultCall[R] => IO[DatabaseError, R] +// Seq[R] dbMultipleResultCall => IO[DatabaseError, Seq[R]] +// FailedOrRow[R] ~ Either[StatusException, Row[R]] - dbSingleResultCallWithStatus => IO[DatabaseError, R] +// Seq[FailedOrRow[R]] ~ Seq[Either[StatusException, Row[R]]] - dbMultipleResultCallWithStatus => IO[DatabaseError, Seq[R]] +// FailedOrRows[R] ~ Either[StatusException, Seq[Row[R]]] - dbMultipleResultCallWithAggregatedStatus => IO[DatabaseError, Seq[R]] + + +sealed trait PaginatedResult[R] +case class ResultHasMore[R](data: Seq[R]) extends PaginatedResult[R] +case class ResultNoMore[R](data: Seq[R]) extends PaginatedResult[R] + trait BaseRepository { - def dbCall[R]( + private def dbCall[R]( dbFuncCall: Task[R], operationName: String ): IO[DatabaseError, R] = { @@ -32,40 +44,22 @@ trait BaseRepository { .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } -// def dbCallWithStatus[R]( -// dbFuncCall: Task[Either[StatusException, R]], -// operationName: String -// ): IO[DatabaseError, R] = { -// dbFuncCall -// .tap { -// case Left(statusException) => -// ZIO.logError( -// s"Exception caused by operation: '$operationName': " + -// s"(${statusException.status.statusCode}), ${statusException.status.statusText}" -// ) -// case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") -// } -// .mapError { -// case statusException: StatusException => -// DatabaseError( -// s"Exception caused by operation: '$operationName': " + -// s"(${statusException.status.statusCode}) ${statusException.status.statusText}" -// ) -// case error => -// DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") -// } -// .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) -// .flatMap { -// case Left(statusException) => fail(DatabaseError( -// s"Failed to execute operation in database: " + -// s"(${statusException.status.statusCode}) ${statusException.status.statusText}" -// )) -// case Right(value) => succeed(value) -// } -// } + def dbSingleResultCall[R]( + dbFuncCall: Task[R], + operationName: String + ): IO[DatabaseError, R] = { + dbCall(dbFuncCall, operationName) + } - def dbCallWithStatus[R]( - dbFuncCall: Task[Either[StatusException, R]], + def dbMultipleResultCall[R]( + dbFuncCall: Task[Seq[R]], + operationName: String + ): IO[DatabaseError, Seq[R]] = { + dbCall(dbFuncCall, operationName) + } + + def dbSingleResultCallWithStatus[R]( + dbFuncCall: Task[FailedOrRow[R]], operationName: String ): IO[DatabaseError, R] = { dbFuncCall @@ -78,12 +72,72 @@ trait BaseRepository { case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") } .flatMap { + case Left(statusException) => ZIO.fail(statusException) + case Right(value) => ZIO.succeed(value.data) + } + .mapError { + case statusException: StatusException => + DatabaseError( + s"Exception caused by operation: '$operationName': " + + s"(${statusException.status.statusCode}) ${statusException.status.statusText}" + ) + case error => + DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") + } + .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) + } + + def dbMultipleResultCallWithAggregatedStatus[R]( + dbFuncCall: Task[FailedOrRows[R]], + operationName: String + ): IO[DatabaseError, Seq[R]] = { + dbFuncCall + .tap { case Left(statusException) => - ZIO.fail(DatabaseError( - s"Failed to execute operation in database: " + + ZIO.logError( + s"Exception caused by operation: '$operationName': " + + s"(${statusException.status.statusCode}), ${statusException.status.statusText}" + ) + case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") + } + .flatMap { + case Left(statusException) => ZIO.fail(statusException) + case Right(value) => ZIO.succeed(value.map(_.data)) + } + .mapError { + case statusException: StatusException => + DatabaseError( + s"Exception caused by operation: '$operationName': " + s"(${statusException.status.statusCode}) ${statusException.status.statusText}" - )) - case Right(value) => ZIO.succeed(value) + ) + case error => + DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") + } + .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) + } + + def dbPaginatedCall[R]( + dbFuncCall: Task[FailedOrRows[R]], + operationName: String + ): IO[DatabaseError, PaginatedResult[R]] = { + dbFuncCall + .tap { + case Left(statusException) => + ZIO.logError( + s"Exception caused by operation: '$operationName': " + + s"(${statusException.status.statusCode}), ${statusException.status.statusText}" + ) + case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") + } + .flatMap { + case Left(statusException) => ZIO.fail(statusException) + case Right(value) => ZIO.succeed( + if (value.nonEmpty && value.head.functionStatus.statusCode == 11){ + ResultHasMore(value.map(_.data)) + } else { + ResultNoMore(value.map(_.data)) + } + ) } .mapError { case statusException: StatusException => diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index 0cc22e5e0..e63d92c9f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -25,7 +25,7 @@ import zio.interop.catz.asyncInstance class CheckpointRepositoryImpl(writeCheckpointFn: WriteCheckpoint) extends CheckpointRepository with BaseRepository { override def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] = { - dbCallWithStatus(writeCheckpointFn(checkpointDTO).map(_.map(x => x.data)), "writeCheckpoint") + dbSingleResultCallWithStatus(writeCheckpointFn(checkpointDTO), "writeCheckpoint") } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala index e18f06ade..0276c744b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala @@ -27,8 +27,7 @@ class FlowRepositoryImpl(getFlowCheckpointsFn: GetFlowCheckpoints) extends FlowR override def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] = { - dbCallWithStatus(getFlowCheckpointsFn(checkpointQueryDTO), "getFlowCheckpoints") - .map(_.map(x => x.data)) + dbMultipleResultCallWithAggregatedStatus(getFlowCheckpointsFn(checkpointQueryDTO), "getFlowCheckpoints") } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index 7d0be1ce2..544fb2cd4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -35,43 +35,33 @@ class PartitioningRepositoryImpl( override def createPartitioningIfNotExists( partitioningSubmitDTO: PartitioningSubmitDTO ): IO[DatabaseError, Unit] = { - dbCallWithStatus(createPartitioningIfNotExistsFn(partitioningSubmitDTO).map(_.map(x => x.data)), "createPartitioningIfNotExists") + dbSingleResultCallWithStatus(createPartitioningIfNotExistsFn(partitioningSubmitDTO), "createPartitioningIfNotExists") } override def createOrUpdateAdditionalData( additionalData: AdditionalDataSubmitDTO ): IO[DatabaseError, Unit] = { - dbCallWithStatus(createOrUpdateAdditionalDataFn(additionalData).map(_.map(x => x.data)), "createOrUpdateAdditionalData") + dbSingleResultCallWithStatus(createOrUpdateAdditionalDataFn(additionalData), "createOrUpdateAdditionalData") } override def getPartitioningMeasures( partitioning: PartitioningDTO ): IO[DatabaseError, Seq[MeasureDTO]] = { - dbCallWithStatus(getPartitioningMeasuresFn(partitioning), "getPartitioningMeasures") - .map(_.map(x => x.data)) + dbMultipleResultCallWithAggregatedStatus(getPartitioningMeasuresFn(partitioning), "getPartitioningMeasures") } -// override def getPartitioningAdditionalData(partitioning: PartitioningDTO): -// IO[DatabaseError, AdditionalDataDTO] = { -// getPartitioningAdditionalDataFn(partitioning) -// .flatMap { -// case Left(value) => ZIO.fail(DatabaseError(s"Failed to retrieve data with status code ${value.status}")) -// case Right(value) => ZIO.succeed(value.map(_.data).toMap) -// }.mapError(error => DatabaseError(error.getMessage)) -// } - override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[DatabaseError, AdditionalDataDTO] = { - dbCallWithStatus( + dbMultipleResultCallWithAggregatedStatus( getPartitioningAdditionalDataFn(partitioning) - .map(_.map(_.map(_.data).toMap)), "getPartitioningAdditionalData" - ) + , "getPartitioningAdditionalData" + ).map(_.toMap) } override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] = { - dbCallWithStatus(getPartitioningCheckpointsFn(checkpointQueryDTO), "getPartitioningCheckpoints") - .map(_.map(x => x.data)) + dbMultipleResultCallWithAggregatedStatus(getPartitioningCheckpointsFn(checkpointQueryDTO), + "getPartitioningCheckpoints") } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 9f4a023e2..bc0450582 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -20,6 +20,7 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.CheckpointService +import za.co.absa.atum.server.model.InternalServerErrorResponse import zio.test.Assertion.failsWithA import zio._ import zio.test._ @@ -46,10 +47,10 @@ object CheckpointControllerUnitTests extends ZIOSpecDefault with TestData { } yield assertTrue(result == checkpointDTO1) }, test("Returns expected InternalServerErrorResponse") { - assertZIO(CheckpointController.createCheckpointV1(checkpointDTO3).exit)(failsWithA[ServiceError]) + assertZIO(CheckpointController.createCheckpointV1(checkpointDTO3).exit)(failsWithA[InternalServerErrorResponse]) }, test("Returns expected GeneralErrorResponse") { - assertZIO(CheckpointController.createCheckpointV1(checkpointDTO2).exit)(failsWithA[ServiceError]) + assertZIO(CheckpointController.createCheckpointV1(checkpointDTO2).exit)(failsWithA[InternalServerErrorResponse]) } ) ).provide( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index 9000e4348..3c5a7c14d 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -27,12 +27,15 @@ import zio.interop.catz.asyncInstance import zio.test.Assertion.{failsWithA, isUnit} import zio.test._ +import za.co.absa.db.fadb.status.Row + object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { private val writeCheckpointMock: WriteCheckpoint = mock(classOf[WriteCheckpoint]) - when(writeCheckpointMock.apply(checkpointDTO1)).thenAnswer(_ => ZIO.succeed(())) - when(writeCheckpointMock.apply(checkpointDTO2)).thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in data")))) + when(writeCheckpointMock.apply(checkpointDTO1)).thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) + when(writeCheckpointMock.apply(checkpointDTO2)) + .thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in data")))) when(writeCheckpointMock.apply(checkpointDTO3)).thenReturn(ZIO.fail(new Exception("boom!"))) private val writeCheckpointMockLayer = ZLayer.succeed(writeCheckpointMock) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/FlowRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/FlowRepositoryUnitTests.scala index d1945d424..ce4e09f71 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/FlowRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/FlowRepositoryUnitTests.scala @@ -17,22 +17,25 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} -import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.database.flows.functions.GetFlowCheckpoints import za.co.absa.atum.server.api.exception.DatabaseError -import za.co.absa.atum.server.model.CheckpointFromDB import zio._ import zio.interop.catz.asyncInstance import zio.test.Assertion.failsWithA import zio.test._ +import za.co.absa.db.fadb.status.{FunctionStatus, Row} object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { private val getFlowCheckpointsMock = mock(classOf[GetFlowCheckpoints]) when(getFlowCheckpointsMock.apply(checkpointQueryDTO1)).thenReturn(ZIO.fail(new Exception("boom!"))) - when(getFlowCheckpointsMock.apply(checkpointQueryDTO2)).thenAnswer(_ => ZIO.succeed(Seq(checkpointFromDB1, checkpointFromDB2))) + when(getFlowCheckpointsMock.apply(checkpointQueryDTO2)) + .thenReturn(ZIO.right(Seq( + Row(FunctionStatus(0, "success"),checkpointFromDB1), + Row(FunctionStatus(0, "success"), checkpointFromDB2))) + ) private val getFlowCheckpointsMockLayer = ZLayer.succeed(getFlowCheckpointsMock) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala index 2b352d303..4902fa259 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala @@ -20,20 +20,22 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.database.runs.functions._ import za.co.absa.atum.server.api.exception.DatabaseError -import za.co.absa.atum.server.model.CheckpointFromDB +//import za.co.absa.atum.server.model.CheckpointFromDB import za.co.absa.db.fadb.exceptions.ErrorInDataException -import za.co.absa.db.fadb.status.FunctionStatus +import za.co.absa.db.fadb.status.{FunctionStatus, Row} import zio._ import zio.interop.catz.asyncInstance -import zio.test.Assertion.{failsWithA, isRight, isUnit} +import zio.test.Assertion.failsWithA import zio.test._ + object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { // Create Partitioning Mocks private val createPartitioningIfNotExistsMock = mock(classOf[CreatePartitioningIfNotExists]) - when(createPartitioningIfNotExistsMock.apply(partitioningSubmitDTO1)).thenAnswer(_ => ZIO.right(())) + when(createPartitioningIfNotExistsMock.apply(partitioningSubmitDTO1)) + .thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) when(createPartitioningIfNotExistsMock.apply(partitioningSubmitDTO2)) .thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in Partitioning data")))) when(createPartitioningIfNotExistsMock.apply(partitioningSubmitDTO3)) @@ -44,7 +46,8 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { // Create Additional Data Mocks private val createOrUpdateAdditionalDataMock = mock(classOf[CreateOrUpdateAdditionalData]) - when(createOrUpdateAdditionalDataMock.apply(additionalDataSubmitDTO1)).thenAnswer(_ => ZIO.right(())) + when(createOrUpdateAdditionalDataMock.apply(additionalDataSubmitDTO1)) + .thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) when(createOrUpdateAdditionalDataMock.apply(additionalDataSubmitDTO2)) .thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in AD data")))) when(createOrUpdateAdditionalDataMock.apply(additionalDataSubmitDTO3)) @@ -55,7 +58,11 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { // Get Partitioning Measures Mocks private val getPartitioningMeasuresMock = mock(classOf[GetPartitioningMeasures]) - when(getPartitioningMeasuresMock.apply(partitioningDTO1)).thenAnswer(_ => ZIO.right(Seq(measureDTO1, measureDTO2))) + when(getPartitioningMeasuresMock.apply(partitioningDTO1)) + .thenReturn(ZIO.right(Seq( + Row(FunctionStatus(0, "success"), measureDTO1), + Row(FunctionStatus(0, "success"), measureDTO2))) + ) when(getPartitioningMeasuresMock.apply(partitioningDTO2)).thenReturn(ZIO.fail(DatabaseError("boom!"))) private val getPartitioningMeasuresMockLayer = ZLayer.succeed(getPartitioningMeasuresMock) @@ -63,7 +70,8 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { // Get Partitioning Additional Data Mocks private val getPartitioningAdditionalDataMock = mock(classOf[GetPartitioningAdditionalData]) - when(getPartitioningAdditionalDataMock.apply(partitioningDTO1)).thenAnswer(_ => ZIO.right(additionalDataDTOSeq1)) + when(getPartitioningAdditionalDataMock.apply(partitioningDTO1)) + .thenReturn(ZIO.right(Seq(Row(FunctionStatus(0, "success"), ("key", Option("value")))))) when(getPartitioningAdditionalDataMock.apply(partitioningDTO2)).thenReturn(ZIO.fail(DatabaseError("boom!"))) private val getPartitioningAdditionalDataMockLayer = ZLayer.succeed(getPartitioningAdditionalDataMock) @@ -71,9 +79,10 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { // Get Partitioning Checkpoints Mocks private val getPartitioningCheckpointsMock = mock(classOf[GetPartitioningCheckpoints]) - when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO1)).thenAnswer(_ => ZIO.succeed(Seq(checkpointFromDB1))) + when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO1)) + .thenReturn(ZIO.right(Seq(Row(FunctionStatus(0, "success"), checkpointFromDB1)))) + when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO3)).thenReturn(ZIO.right(Seq.empty)) when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO2)).thenReturn(ZIO.fail(DatabaseError("boom!"))) - when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO3)).thenAnswer(_ => ZIO.succeed(Seq.empty)) private val getPartitioningCheckpointsMockLayer = ZLayer.succeed(getPartitioningCheckpointsMock) @@ -84,16 +93,16 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- PartitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO1) - } yield assert(result)(isUnit) + } yield assertTrue(result == ()) }, test("Returns expected Left with StatusException") { for { - result <- PartitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO2) - } yield assert(result)(isUnit) + result <- PartitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO2).exit + } yield assertTrue(result == Exit.fail(DatabaseError("error in Partitioning data"))) }, test("Returns expected DatabaseError") { assertZIO(PartitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO3).exit)( - failsWithA[DatabaseError] + failsWithA[Exception] ) } ), @@ -102,16 +111,16 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- PartitioningRepository.createOrUpdateAdditionalData(additionalDataSubmitDTO1) - } yield assert(result)(isUnit) + } yield assertTrue(result == ()) }, test("Returns expected Left with StatusException") { for { - result <- PartitioningRepository.createOrUpdateAdditionalData(additionalDataSubmitDTO2) - } yield assert(result)(isUnit) + result <- PartitioningRepository.createOrUpdateAdditionalData(additionalDataSubmitDTO2).exit + } yield assertTrue(result == Exit.fail(DatabaseError("error in AD data"))) }, test("Returns expected DatabaseError") { assertZIO(PartitioningRepository.createOrUpdateAdditionalData(additionalDataSubmitDTO3).exit)( - failsWithA[DatabaseError] + failsWithA[Exception] ) } ), @@ -120,7 +129,10 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Seq") { for { result <- PartitioningRepository.getPartitioningMeasures(partitioningDTO1) - } yield assertTrue(result == Seq(measureDTO1, measureDTO2)) + } yield assertTrue(result == Seq( + Row(FunctionStatus(0, "success"), measureDTO1), + Row(FunctionStatus(0, "success"), measureDTO2) + )) }, test("Returns expected Exception") { assertZIO(PartitioningRepository.getPartitioningMeasures(partitioningDTO2).exit)( @@ -133,7 +145,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with empty Map") { for { result <- PartitioningRepository.getPartitioningAdditionalData(partitioningDTO1) - } yield assertTrue(result == additionalDataDTO1) + } yield assertTrue(result.toSeq == Seq(Row(FunctionStatus(0, "success"), ("key", Option("value"))))) }, test("Returns expected Left with DatabaseError") { assertZIO(PartitioningRepository.getPartitioningAdditionalData(partitioningDTO2).exit)( @@ -146,7 +158,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Seq") { for { result <- PartitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO1) - } yield assertTrue(result == Seq(checkpointFromDB1)) + } yield assertTrue(result == Seq(Row(FunctionStatus(0, "success"), checkpointFromDB1))) }, test("Returns expected DatabaseError") { assertZIO(PartitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO2).exit)( @@ -156,7 +168,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Seq.empty") { for { result <- PartitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO3) - } yield assertTrue(result.isInstanceOf[Seq[CheckpointFromDB]] && result.isEmpty) + } yield assertTrue(result.isInstanceOf[Seq[Row[FunctionStatus]]] && result.isEmpty) } ) ).provide( From ee594e44ad00cc61f1345fe0a08515b4429b5dab Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 26 Jul 2024 09:19:47 +0200 Subject: [PATCH 15/31] Fixing PartitioningRepositoryUnitTests --- .../repository/PartitioningRepositoryUnitTests.scala | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala index 4902fa259..a0061867b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala @@ -129,10 +129,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Seq") { for { result <- PartitioningRepository.getPartitioningMeasures(partitioningDTO1) - } yield assertTrue(result == Seq( - Row(FunctionStatus(0, "success"), measureDTO1), - Row(FunctionStatus(0, "success"), measureDTO2) - )) + } yield assertTrue(result == Seq(measureDTO1, measureDTO2)) }, test("Returns expected Exception") { assertZIO(PartitioningRepository.getPartitioningMeasures(partitioningDTO2).exit)( @@ -145,7 +142,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with empty Map") { for { result <- PartitioningRepository.getPartitioningAdditionalData(partitioningDTO1) - } yield assertTrue(result.toSeq == Seq(Row(FunctionStatus(0, "success"), ("key", Option("value"))))) + } yield assertTrue(result.toSeq == Seq(("key", Option("value")))) }, test("Returns expected Left with DatabaseError") { assertZIO(PartitioningRepository.getPartitioningAdditionalData(partitioningDTO2).exit)( @@ -158,7 +155,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Seq") { for { result <- PartitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO1) - } yield assertTrue(result == Seq(Row(FunctionStatus(0, "success"), checkpointFromDB1))) + } yield assertTrue(result == Seq(checkpointFromDB1)) }, test("Returns expected DatabaseError") { assertZIO(PartitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO2).exit)( @@ -168,7 +165,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Seq.empty") { for { result <- PartitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO3) - } yield assertTrue(result.isInstanceOf[Seq[Row[FunctionStatus]]] && result.isEmpty) + } yield assertTrue(result.isEmpty) } ) ).provide( From d684420f3309416fa273af96b536b9b29d20db0d Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 26 Jul 2024 09:24:47 +0200 Subject: [PATCH 16/31] Fixed PartitioningRepositoryUnitTests --- .../api/repository/PartitioningRepositoryUnitTests.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala index a0061867b..107b057f9 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala @@ -20,7 +20,6 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.database.runs.functions._ import za.co.absa.atum.server.api.exception.DatabaseError -//import za.co.absa.atum.server.model.CheckpointFromDB import za.co.absa.db.fadb.exceptions.ErrorInDataException import za.co.absa.db.fadb.status.{FunctionStatus, Row} import zio._ @@ -98,11 +97,11 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Left with StatusException") { for { result <- PartitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO2).exit - } yield assertTrue(result == Exit.fail(DatabaseError("error in Partitioning data"))) + } yield assertTrue(result == Exit.fail(DatabaseError("Exception caused by operation: 'createPartitioningIfNotExists': (50) error in Partitioning data"))) }, test("Returns expected DatabaseError") { assertZIO(PartitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO3).exit)( - failsWithA[Exception] + failsWithA[DatabaseError] ) } ), @@ -116,11 +115,11 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Left with StatusException") { for { result <- PartitioningRepository.createOrUpdateAdditionalData(additionalDataSubmitDTO2).exit - } yield assertTrue(result == Exit.fail(DatabaseError("error in AD data"))) + } yield assertTrue(result == Exit.fail(DatabaseError("Exception caused by operation: 'createOrUpdateAdditionalData': (50) error in AD data"))) }, test("Returns expected DatabaseError") { assertZIO(PartitioningRepository.createOrUpdateAdditionalData(additionalDataSubmitDTO3).exit)( - failsWithA[Exception] + failsWithA[DatabaseError] ) } ), From 1908297351f641678e380e99164632be410478ea Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 26 Jul 2024 09:39:00 +0200 Subject: [PATCH 17/31] Fixed WriteCheckpointRepositoryUnitTests --- .../api/repository/CheckpointRepositoryUnitTests.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index 3c5a7c14d..7017056ca 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -35,7 +35,7 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { when(writeCheckpointMock.apply(checkpointDTO1)).thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) when(writeCheckpointMock.apply(checkpointDTO2)) - .thenReturn(ZIO.left(ErrorInDataException(FunctionStatus(50, "error in data")))) + .thenReturn(ZIO.fail(DatabaseError("Operation 'writeCheckpoint' failed with unexpected error: null"))) when(writeCheckpointMock.apply(checkpointDTO3)).thenReturn(ZIO.fail(new Exception("boom!"))) private val writeCheckpointMockLayer = ZLayer.succeed(writeCheckpointMock) @@ -47,12 +47,12 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- CheckpointRepository.writeCheckpoint(checkpointDTO1) - } yield assert(result)(isUnit) + } yield assertTrue(result == ()) }, test("Returns expected Left with StatusException") { for { - result <- CheckpointRepository.writeCheckpoint(checkpointDTO2) - } yield assert(result)(isUnit) + result <- CheckpointRepository.writeCheckpoint(checkpointDTO2).exit + } yield assertTrue(result == Exit.fail(DatabaseError("Operation 'writeCheckpoint' failed with unexpected error: null"))) }, test("Returns expected DatabaseError") { assertZIO(CheckpointRepository.writeCheckpoint(checkpointDTO3).exit)(failsWithA[DatabaseError]) From fa6e07654b4cfdf7a1c29420466206846e931ba2 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 26 Jul 2024 10:22:49 +0200 Subject: [PATCH 18/31] Implement defaultErrorHandler method --- .../api/repository/BaseRepository.scala | 80 +++++++++++-------- .../PartitioningRepositoryImpl.scala | 13 ++- 2 files changed, 57 insertions(+), 36 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index 8990a6f74..7e6605372 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -27,13 +27,43 @@ import zio._ // Seq[FailedOrRow[R]] ~ Seq[Either[StatusException, Row[R]]] - dbMultipleResultCallWithStatus => IO[DatabaseError, Seq[R]] // FailedOrRows[R] ~ Either[StatusException, Seq[Row[R]]] - dbMultipleResultCallWithAggregatedStatus => IO[DatabaseError, Seq[R]] - sealed trait PaginatedResult[R] case class ResultHasMore[R](data: Seq[R]) extends PaginatedResult[R] case class ResultNoMore[R](data: Seq[R]) extends PaginatedResult[R] trait BaseRepository { + private def dbCall[R, T]( + dbFuncCall: Task[T], + operationName: String, + successHandler: T => IO[DatabaseError, R], + errorHandler: PartialFunction[Throwable, DatabaseError] + ): IO[DatabaseError, R] = { + dbFuncCall + .tap { + case Left(statusException: StatusException) => + ZIO.logError( + s"Exception caused by operation: '$operationName': " + + s"(${statusException.status.statusCode}), ${statusException.status.statusText}" + ) + case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") + case _ => ZIO.logDebug(s"Operation '$operationName' succeeded in database") + } + .flatMap(successHandler) + .mapError(errorHandler) + .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) + } + + private def defaultErrorHandler[R](operationName: String): PartialFunction[Throwable, DatabaseError] = { + case statusException: StatusException => + DatabaseError( + s"Exception caused by operation: '$operationName': " + + s"(${statusException.status.statusCode}) ${statusException.status.statusText}" + ) + case error => + DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") + } + private def dbCall[R]( dbFuncCall: Task[R], operationName: String @@ -45,16 +75,16 @@ trait BaseRepository { } def dbSingleResultCall[R]( - dbFuncCall: Task[R], - operationName: String - ): IO[DatabaseError, R] = { + dbFuncCall: Task[R], + operationName: String + ): IO[DatabaseError, R] = { dbCall(dbFuncCall, operationName) } def dbMultipleResultCall[R]( - dbFuncCall: Task[Seq[R]], - operationName: String - ): IO[DatabaseError, Seq[R]] = { + dbFuncCall: Task[Seq[R]], + operationName: String + ): IO[DatabaseError, Seq[R]] = { dbCall(dbFuncCall, operationName) } @@ -76,21 +106,15 @@ trait BaseRepository { case Right(value) => ZIO.succeed(value.data) } .mapError { - case statusException: StatusException => - DatabaseError( - s"Exception caused by operation: '$operationName': " + - s"(${statusException.status.statusCode}) ${statusException.status.statusText}" - ) - case error => - DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") + defaultErrorHandler(operationName) } .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } def dbMultipleResultCallWithAggregatedStatus[R]( - dbFuncCall: Task[FailedOrRows[R]], - operationName: String - ): IO[DatabaseError, Seq[R]] = { + dbFuncCall: Task[FailedOrRows[R]], + operationName: String + ): IO[DatabaseError, Seq[R]] = { dbFuncCall .tap { case Left(statusException) => @@ -105,21 +129,15 @@ trait BaseRepository { case Right(value) => ZIO.succeed(value.map(_.data)) } .mapError { - case statusException: StatusException => - DatabaseError( - s"Exception caused by operation: '$operationName': " + - s"(${statusException.status.statusCode}) ${statusException.status.statusText}" - ) - case error => - DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") + defaultErrorHandler(operationName) } .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } def dbPaginatedCall[R]( - dbFuncCall: Task[FailedOrRows[R]], - operationName: String - ): IO[DatabaseError, PaginatedResult[R]] = { + dbFuncCall: Task[FailedOrRows[R]], + operationName: String + ): IO[DatabaseError, PaginatedResult[R]] = { dbFuncCall .tap { case Left(statusException) => @@ -140,13 +158,7 @@ trait BaseRepository { ) } .mapError { - case statusException: StatusException => - DatabaseError( - s"Exception caused by operation: '$operationName': " + - s"(${statusException.status.statusCode}) ${statusException.status.statusText}" - ) - case error => - DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") + defaultErrorHandler(operationName) } .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index 544fb2cd4..2e98447e6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -16,8 +16,17 @@ package za.co.absa.atum.server.api.repository -import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} -import za.co.absa.atum.server.api.database.runs.functions.{CreateOrUpdateAdditionalData, CreatePartitioningIfNotExists, GetPartitioningAdditionalData, GetPartitioningCheckpoints, GetPartitioningMeasures} +import za.co.absa.atum.model.dto.{ + AdditionalDataDTO, + AdditionalDataSubmitDTO, + CheckpointQueryDTO, MeasureDTO, + PartitioningDTO, PartitioningSubmitDTO} +import za.co.absa.atum.server.api.database.runs.functions.{ + CreateOrUpdateAdditionalData, + CreatePartitioningIfNotExists, + GetPartitioningAdditionalData, + GetPartitioningCheckpoints, + GetPartitioningMeasures} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB import zio._ From d51b81d44d91e4f97064f653dabdbef08e56b1e3 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 26 Jul 2024 11:30:47 +0200 Subject: [PATCH 19/31] optimising partitioningRepository base repository --- .../api/repository/BaseRepository.scala | 95 +++++-------------- 1 file changed, 26 insertions(+), 69 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index 7e6605372..a7ca1cb0b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -33,28 +33,21 @@ case class ResultNoMore[R](data: Seq[R]) extends PaginatedResult[R] trait BaseRepository { - private def dbCall[R, T]( - dbFuncCall: Task[T], - operationName: String, - successHandler: T => IO[DatabaseError, R], - errorHandler: PartialFunction[Throwable, DatabaseError] - ): IO[DatabaseError, R] = { + private def logOperationResult[R](operationName: String, dbFuncCall: Task[R]): + ZIO[Any, Throwable, R] = { dbFuncCall .tap { - case Left(statusException: StatusException) => - ZIO.logError( - s"Exception caused by operation: '$operationName': " + - s"(${statusException.status.statusCode}), ${statusException.status.statusText}" - ) - case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") - case _ => ZIO.logDebug(s"Operation '$operationName' succeeded in database") - } - .flatMap(successHandler) - .mapError(errorHandler) - .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) + case Left(statusException: StatusException) => + ZIO.logError( + s"Exception caused by operation: '$operationName': " + + s"(${statusException.status.statusCode}), ${statusException.status.statusText}" + ) + case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") + } } - private def defaultErrorHandler[R](operationName: String): PartialFunction[Throwable, DatabaseError] = { + private def defaultErrorHandler[R](operationName: String): + PartialFunction[Throwable, DatabaseError] = { case statusException: StatusException => DatabaseError( s"Exception caused by operation: '$operationName': " + @@ -64,43 +57,27 @@ trait BaseRepository { DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") } - private def dbCall[R]( - dbFuncCall: Task[R], - operationName: String - ): IO[DatabaseError, R] = { + private def dbCall[R](dbFuncCall: Task[R], operationName: String): + IO[DatabaseError, R] = { dbFuncCall .zipLeft(ZIO.logDebug(s"Operation '$operationName' succeeded in database")) .mapError(error => DatabaseError(error.getMessage)) .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } - def dbSingleResultCall[R]( - dbFuncCall: Task[R], - operationName: String - ): IO[DatabaseError, R] = { + def dbSingleResultCall[R](dbFuncCall: Task[R], operationName: String): + IO[DatabaseError, R] = { dbCall(dbFuncCall, operationName) } - def dbMultipleResultCall[R]( - dbFuncCall: Task[Seq[R]], - operationName: String - ): IO[DatabaseError, Seq[R]] = { + def dbMultipleResultCall[R](dbFuncCall: Task[Seq[R]], operationName: String): + IO[DatabaseError, Seq[R]] = { dbCall(dbFuncCall, operationName) } - def dbSingleResultCallWithStatus[R]( - dbFuncCall: Task[FailedOrRow[R]], - operationName: String - ): IO[DatabaseError, R] = { - dbFuncCall - .tap { - case Left(statusException) => - ZIO.logError( - s"Exception caused by operation: '$operationName': " + - s"(${statusException.status.statusCode}), ${statusException.status.statusText}" - ) - case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") - } + def dbSingleResultCallWithStatus[R](dbFuncCall: Task[FailedOrRow[R]], operationName: String): + IO[DatabaseError, R] = { + logOperationResult(operationName, dbFuncCall) .flatMap { case Left(statusException) => ZIO.fail(statusException) case Right(value) => ZIO.succeed(value.data) @@ -111,19 +88,9 @@ trait BaseRepository { .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } - def dbMultipleResultCallWithAggregatedStatus[R]( - dbFuncCall: Task[FailedOrRows[R]], - operationName: String - ): IO[DatabaseError, Seq[R]] = { - dbFuncCall - .tap { - case Left(statusException) => - ZIO.logError( - s"Exception caused by operation: '$operationName': " + - s"(${statusException.status.statusCode}), ${statusException.status.statusText}" - ) - case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") - } + def dbMultipleResultCallWithAggregatedStatus[R](dbFuncCall: Task[FailedOrRows[R]], operationName: String): + IO[DatabaseError, Seq[R]] = { + logOperationResult(operationName, dbFuncCall) .flatMap { case Left(statusException) => ZIO.fail(statusException) case Right(value) => ZIO.succeed(value.map(_.data)) @@ -134,19 +101,9 @@ trait BaseRepository { .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } - def dbPaginatedCall[R]( - dbFuncCall: Task[FailedOrRows[R]], - operationName: String - ): IO[DatabaseError, PaginatedResult[R]] = { - dbFuncCall - .tap { - case Left(statusException) => - ZIO.logError( - s"Exception caused by operation: '$operationName': " + - s"(${statusException.status.statusCode}), ${statusException.status.statusText}" - ) - case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") - } + def dbPaginatedCall[R](dbFuncCall: Task[FailedOrRows[R]], operationName: String): + IO[DatabaseError, PaginatedResult[R]] = { + logOperationResult(operationName, dbFuncCall) .flatMap { case Left(statusException) => ZIO.fail(statusException) case Right(value) => ZIO.succeed( From bff113b9b38f1f1d19f6194d9a379ce922f9aac3 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 26 Jul 2024 11:46:54 +0200 Subject: [PATCH 20/31] Fixing CheckpointServiceUnitTests --- .../api/service/CheckpointServiceUnitTests.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala index 7d730b655..14c0db216 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala @@ -30,8 +30,9 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { private val checkpointRepositoryMock = mock(classOf[CheckpointRepository]) - when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO1)).thenAnswer(_ => ZIO.right(())) - when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO2)).thenReturn(ZIO.fail(DatabaseError("error in data"))) + when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO1)).thenReturn(ZIO.succeed(())) + when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO2)). + thenReturn(ZIO.fail(DatabaseError("error in data"))) when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO3)).thenReturn(ZIO.fail(DatabaseError("boom!"))) private val checkpointRepositoryMockLayer = ZLayer.succeed(checkpointRepositoryMock) @@ -43,12 +44,12 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- CheckpointService.saveCheckpoint(checkpointDTO1) - } yield assert(result)(isUnit) + } yield assertTrue(result == ()) }, test("Returns expected Left with StatusException") { for { - result <- CheckpointService.saveCheckpoint(checkpointDTO2) - } yield assert(result)(isUnit) + result <- CheckpointService.saveCheckpoint(checkpointDTO2).exit + } yield assertTrue(result == Exit.fail(ServiceError("Failed to perform 'saveCheckpoint': error in data"))) }, test("Returns expected ServiceError") { assertZIO(CheckpointService.saveCheckpoint(checkpointDTO3).exit)(failsWithA[ServiceError]) From dce3de71d17940e7fdf2cbf9f174cf9f991805fd Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 26 Jul 2024 11:54:19 +0200 Subject: [PATCH 21/31] Fixing PartitioningServiceUnitTests --- .../api/service/PartitioningServiceUnitTests.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala index 341a476b5..14b2e2575 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala @@ -68,8 +68,8 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { }, test("Returns expected Left with StatusException") { for { - result <- PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO2) - } yield assert(result)(isUnit) + result <- PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO2).exit + } yield assertTrue(result == Exit.fail(ServiceError("Failed to perform 'createPartitioningIfNotExists': error in data"))) }, test("Returns expected ServiceError") { assertZIO(PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO3).exit)( @@ -81,12 +81,12 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO1) - } yield assert(result)(isUnit) + } yield assertTrue(result == ()) }, test("Returns expected Left with StatusException") { for { - result <- PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO2) - } yield assert(result)(isUnit) + result <- PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO2).exit + } yield assertTrue(result == Exit.fail(ServiceError("Failed to perform 'createOrUpdateAdditionalData': error in AD data"))) }, test("Returns expected ServiceError") { assertZIO(PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO3).exit)( From d53a58ae02de708a84fa68062f789eadde5891e3 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 26 Jul 2024 11:55:39 +0200 Subject: [PATCH 22/31] remove Unit and unused imports from PartitioningServiceUnitTests --- .../server/api/service/PartitioningServiceUnitTests.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala index 14b2e2575..8dd6a22b4 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala @@ -20,7 +20,7 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.{DatabaseError, ServiceError} import za.co.absa.atum.server.api.repository.PartitioningRepository -import zio.test.Assertion.{equalTo, failsWithA, isLeft, isRight, isUnit} +import zio.test.Assertion.failsWithA import zio.test._ import zio._ @@ -64,7 +64,7 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Unit") { for { result <- PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO1) - } yield assert(result)(isUnit) + } yield assertTrue(result == ()) }, test("Returns expected Left with StatusException") { for { From 1a4bfb1e3c6c9635c83c9e0ad658456be02f20ba Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 26 Jul 2024 11:56:49 +0200 Subject: [PATCH 23/31] remove Unit and unused imports from CheckpointServiceUnitTests --- .../atum/server/api/service/CheckpointServiceUnitTests.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala index 14c0db216..63ad67413 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala @@ -20,9 +20,7 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.{DatabaseError, ServiceError} import za.co.absa.atum.server.api.repository.CheckpointRepository -import za.co.absa.db.fadb.exceptions.ErrorInDataException -import za.co.absa.db.fadb.status.FunctionStatus -import zio.test.Assertion.{failsWithA, isLeft, isRight, isUnit} +import zio.test.Assertion.failsWithA import zio.test._ import zio._ From 6b90db11ed9c1b1a57ae23481380db4d02916cce Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 30 Jul 2024 11:47:58 +0200 Subject: [PATCH 24/31] Adding AdditionalDataFromDB, upgrading fa-db to 5, and fixing bugs --- project/Dependencies.scala | 2 +- .../flows/functions/GetFlowCheckpoints.scala | 16 +++++++++++++++- .../GetPartitioningAdditionalData.scala | 8 +++++--- .../functions/GetPartitioningCheckpoints.scala | 16 +++++++++++++++- .../runs/functions/GetPartitioningMeasures.scala | 6 +++++- .../repository/PartitioningRepositoryImpl.scala | 5 ++--- .../PartitioningRepositoryUnitTests.scala | 7 ++++--- 7 files changed, 47 insertions(+), 13 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 0a003bdb7..687226999 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -42,7 +42,7 @@ object Dependencies { val postgresql = "42.6.0" - val fadb = "0.4.1" + val fadb = "0.5.0" val logback = "1.2.3" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 2c479af69..4cfac309f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -41,7 +41,21 @@ class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task] fr"${values.limit}", fr"${values.checkpointName}" ) - ) with StandardStatusHandling with ByFirstErrorStatusAggregator + ) with StandardStatusHandling with ByFirstErrorStatusAggregator { + override val fieldsToSelect: Seq[String] = Seq( + "status", + "status_text", + "id_checkpoint", + "checkpoint_name", + "author", + "measured_by_atum_agent", + "measure_name", + "measured_columns", + "measurement_value", + "checkpoint_start_time", + "checkpoint_end_time" + ) +} object GetFlowCheckpoints { val layer: URLayer[PostgresDatabaseProvider, GetFlowCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala index ae06790bd..6392f7e25 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala @@ -22,7 +22,7 @@ import doobie.util.Read import za.co.absa.atum.model.dto.PartitioningDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs -import za.co.absa.atum.server.model.PartitioningForDB +import za.co.absa.atum.server.model.{AdditionalDataFromDB, PartitioningForDB} import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWithAggStatus import za.co.absa.db.fadb.doobie.DoobieEngine @@ -35,9 +35,11 @@ import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusA import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling class GetPartitioningAdditionalData (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, (String, Option[String]), Task]( + extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, AdditionalDataFromDB, Task]( values => Seq(fr"${PartitioningForDB.fromSeqPartitionDTO(values).asJson}")) - with StandardStatusHandling with ByFirstErrorStatusAggregator + with StandardStatusHandling with ByFirstErrorStatusAggregator { + override val fieldsToSelect: Seq[String] = Seq("status", "status_text", "ad_name", "ad_value") +} object GetPartitioningAdditionalData { val layer: URLayer[PostgresDatabaseProvider, GetPartitioningAdditionalData] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 701ef73ec..57b514c54 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -43,7 +43,21 @@ class GetPartitioningCheckpoints (implicit schema: DBSchema, dbEngine: DoobieEng fr"${values.limit}", fr"${values.checkpointName}" ) - ) with StandardStatusHandling with ByFirstErrorStatusAggregator + ) with StandardStatusHandling with ByFirstErrorStatusAggregator { + override val fieldsToSelect: Seq[String] = Seq( + "status", + "status_text", + "id_checkpoint", + "checkpoint_name", + "author", + "measured_by_atum_agent", + "measure_name", + "measured_columns", + "measurement_value", + "checkpoint_start_time", + "checkpoint_end_time" + ) +} object GetPartitioningCheckpoints { val layer: URLayer[PostgresDatabaseProvider, GetPartitioningCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala index 9c7e1ac6b..0089070e0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala @@ -37,7 +37,11 @@ import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling class GetPartitioningMeasures (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, MeasureDTO, Task]( values => Seq(fr"${PartitioningForDB.fromSeqPartitionDTO(values).asJson}")) - with StandardStatusHandling with ByFirstErrorStatusAggregator + with StandardStatusHandling with ByFirstErrorStatusAggregator { + + override val fieldsToSelect: Seq[String] = Seq("status", "status_text", "measure_name", "measured_columns") + +} object GetPartitioningMeasures { val layer: URLayer[PostgresDatabaseProvider, GetPartitioningMeasures] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index 2e98447e6..a7a0e9627 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -31,6 +31,7 @@ import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB import zio._ import zio.interop.catz.asyncInstance +import za.co.absa.atum.server.model.AdditionalDataFromDB class PartitioningRepositoryImpl( @@ -63,17 +64,15 @@ class PartitioningRepositoryImpl( dbMultipleResultCallWithAggregatedStatus( getPartitioningAdditionalDataFn(partitioning) , "getPartitioningAdditionalData" - ).map(_.toMap) + ).map( _.map { case AdditionalDataFromDB(adName, adValue) => adName.get -> adValue }.toMap) } - override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] = { dbMultipleResultCallWithAggregatedStatus(getPartitioningCheckpointsFn(checkpointQueryDTO), "getPartitioningCheckpoints") } - } object PartitioningRepositoryImpl { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala index 107b057f9..9992b1248 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala @@ -26,6 +26,7 @@ import zio._ import zio.interop.catz.asyncInstance import zio.test.Assertion.failsWithA import zio.test._ +import za.co.absa.atum.server.model.AdditionalDataFromDB object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { @@ -70,7 +71,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { private val getPartitioningAdditionalDataMock = mock(classOf[GetPartitioningAdditionalData]) when(getPartitioningAdditionalDataMock.apply(partitioningDTO1)) - .thenReturn(ZIO.right(Seq(Row(FunctionStatus(0, "success"), ("key", Option("value")))))) + .thenReturn(ZIO.right(Seq(Row(FunctionStatus(0, "success"), AdditionalDataFromDB(Some("key"), Some("value")))))) when(getPartitioningAdditionalDataMock.apply(partitioningDTO2)).thenReturn(ZIO.fail(DatabaseError("boom!"))) private val getPartitioningAdditionalDataMockLayer = ZLayer.succeed(getPartitioningAdditionalDataMock) @@ -138,10 +139,10 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { ), suite("GetPartitioningAdditionalDataSuite")( - test("Returns expected Right with empty Map") { + test("Returns expected Right with Map") { for { result <- PartitioningRepository.getPartitioningAdditionalData(partitioningDTO1) - } yield assertTrue(result.toSeq == Seq(("key", Option("value")))) + } yield assertTrue(result.get("key").contains(Some("value")) && result.size == 1) }, test("Returns expected Left with DatabaseError") { assertZIO(PartitioningRepository.getPartitioningAdditionalData(partitioningDTO2).exit)( From 9292faae354619e2606b8c4a60a4135d8c2da7f4 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 30 Jul 2024 11:48:28 +0200 Subject: [PATCH 25/31] Adding AdditionalDataFromDB, upgrading fa-db to 5, and fixing bugs --- .../za/co/absa/atum/server/model/AdditionalDataFromDB.scala | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 server/src/main/scala/za/co/absa/atum/server/model/AdditionalDataFromDB.scala diff --git a/server/src/main/scala/za/co/absa/atum/server/model/AdditionalDataFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/AdditionalDataFromDB.scala new file mode 100644 index 000000000..48a8dbb94 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/model/AdditionalDataFromDB.scala @@ -0,0 +1,6 @@ +package za.co.absa.atum.server.model + +case class AdditionalDataFromDB ( + adName: Option[String], + adValue: Option[String] +) From e26ce126cd798df217710d3180f0e33e1700b62b Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 30 Jul 2024 12:15:10 +0200 Subject: [PATCH 26/31] Fixing test cases --- .../functions/CreateOrUpdateAdditionalData.scala | 4 +--- .../CreatePartitioningIfNotExists.scala | 3 --- .../GetPartitioningAdditionalData.scala | 2 -- .../functions/GetPartitioningCheckpoints.scala | 2 -- .../runs/functions/GetPartitioningMeasures.scala | 2 -- .../runs/functions/WriteCheckpoint.scala | 1 - .../atum/server/model/AdditionalDataFromDB.scala | 16 ++++++++++++++++ ...titioningAdditionalDataIntegrationTests.scala | 7 ++++--- ...GetPartitioningMeasuresIntegrationTests.scala | 2 +- 9 files changed, 22 insertions(+), 17 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index 793201241..948232fd6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -16,16 +16,14 @@ package za.co.absa.atum.server.api.database.runs.functions -import doobie.Fragment import doobie.implicits.toSqlInterpolator -import doobie.util.Read import za.co.absa.atum.model.dto.AdditionalDataSubmitDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus -import za.co.absa.db.fadb.doobie.{DoobieEngine, StatusWithData} +import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import zio._ import zio.interop.catz._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index fe8a478c3..2a384ad0e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -16,10 +16,7 @@ package za.co.absa.atum.server.api.database.runs.functions - -import doobie.Fragment import doobie.implicits.toSqlInterpolator -import doobie.util.Read import za.co.absa.atum.model.dto.PartitioningSubmitDTO import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.db.fadb.DBSchema diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala index 6392f7e25..098fad56c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala @@ -16,9 +16,7 @@ package za.co.absa.atum.server.api.database.runs.functions -import doobie.Fragment import doobie.implicits.toSqlInterpolator -import doobie.util.Read import za.co.absa.atum.model.dto.PartitioningDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 57b514c54..51c296c90 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -16,9 +16,7 @@ package za.co.absa.atum.server.api.database.runs.functions -import doobie.Fragment import doobie.implicits.toSqlInterpolator -import doobie.util.Read import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala index 0089070e0..96d4581db 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala @@ -16,9 +16,7 @@ package za.co.absa.atum.server.api.database.runs.functions -import doobie.Fragment import doobie.implicits.toSqlInterpolator -import doobie.util.Read import za.co.absa.atum.model.dto.{MeasureDTO, PartitioningDTO} import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.db.fadb.DBSchema diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 3f64e6284..0398c96b5 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.server.api.database.runs.functions -import doobie.Fragment import doobie.implicits.toSqlInterpolator import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.model.PartitioningForDB diff --git a/server/src/main/scala/za/co/absa/atum/server/model/AdditionalDataFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/AdditionalDataFromDB.scala index 48a8dbb94..c22c8a4aa 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/AdditionalDataFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/AdditionalDataFromDB.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.server.model case class AdditionalDataFromDB ( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalDataIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalDataIntegrationTests.scala index 133324989..abdd8eca3 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalDataIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalDataIntegrationTests.scala @@ -20,10 +20,11 @@ import za.co.absa.atum.model.dto.{PartitionDTO, PartitioningDTO} import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import za.co.absa.db.fadb.exceptions.DataNotFoundException +import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.interop.catz.asyncInstance import zio.test._ -import zio.test.Assertion._ object GetPartitioningAdditionalDataIntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -32,8 +33,8 @@ object GetPartitioningAdditionalDataIntegrationTests extends ConfigProviderTest val partitioningDTO: PartitioningDTO = Seq(PartitionDTO("stringA", "stringB"), PartitionDTO("string2", "string2")) for { getPartitioningAdditionalData <- ZIO.service[GetPartitioningAdditionalData] - exit <- getPartitioningAdditionalData(partitioningDTO).exit - } yield assert(exit)(failsWithA[doobie.util.invariant.NonNullableColumnRead]) + result <- getPartitioningAdditionalData(partitioningDTO) + } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) } ).provide( GetPartitioningAdditionalData.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala index aedc0e249..1ae885596 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala @@ -31,7 +31,7 @@ object GetPartitioningMeasuresIntegrationTests extends ConfigProviderTest { suite("GetPartitioningMeasuresSuite")( test("Returns expected sequence of Measures with existing partitioning") { - val partitioningDTO: PartitioningDTO = Seq(PartitionDTO("string1", "string1"), PartitionDTO("string2", "string2")) + val partitioningDTO: PartitioningDTO = Seq(PartitionDTO("string11", "string11"), PartitionDTO("string12", "string12")) for { getPartitioningMeasures <- ZIO.service[GetPartitioningMeasures] result <- getPartitioningMeasures(partitioningDTO).exit From bb2b202059aa8b40310ad68266a4eb86481c5d85 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 1 Aug 2024 03:14:00 +0200 Subject: [PATCH 27/31] Addressing GitHub comments --- .../za/co/absa/atum/model/dto/package.scala | 1 - .../api/controller/BaseController.scala | 21 +---- .../server/api/database/DoobieImplicits.scala | 81 +------------------ .../flows/functions/GetFlowCheckpoints.scala | 4 +- .../CreateOrUpdateAdditionalData.scala | 3 +- .../CreatePartitioningIfNotExists.scala | 5 +- .../GetPartitioningAdditionalData.scala | 3 +- .../GetPartitioningCheckpoints.scala | 6 +- .../runs/functions/WriteCheckpoint.scala | 6 +- .../api/repository/BaseRepository.scala | 74 +++-------------- .../api/repository/FlowRepository.scala | 3 +- .../api/repository/FlowRepositoryImpl.scala | 3 +- .../repository/PartitioningRepository.scala | 20 ++--- .../PartitioningRepositoryImpl.scala | 15 +--- .../atum/server/api/service/BaseService.scala | 11 +-- .../atum/server/api/service/FlowService.scala | 4 +- .../api/service/PartitioningService.scala | 15 ++-- .../api/service/PartitioningServiceImpl.scala | 14 +--- .../GetFlowCheckpointsIntegrationTests.scala | 42 ++++++++++ 19 files changed, 89 insertions(+), 242 deletions(-) create mode 100644 server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala index b89d71409..2d3952241 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala @@ -17,7 +17,6 @@ package za.co.absa.atum.model -import io.circe.generic.semiauto._ import io.circe._ package object dto { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index 6b12e1ef9..a4aeac9dc 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -17,9 +17,8 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.server.api.exception.ServiceError -import za.co.absa.atum.server.model.{ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{ErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} -import za.co.absa.db.fadb.exceptions.StatusException import zio._ trait BaseController { @@ -39,24 +38,6 @@ trait BaseController { } - def serviceCallWithStatus[A, B]( - serviceCall: IO[ServiceError, Either[StatusException, A]], - onSuccessFnc: A => B - ): IO[ErrorResponse, B] = { - - serviceCall - .mapError { serviceError: ServiceError => - InternalServerErrorResponse(serviceError.message) - } - .flatMap { - case Left(statusException) => - ZIO.fail(GeneralErrorResponse(s"(${statusException.status.statusCode}) ${statusException.status.statusText}")) - case Right(result) => - ZIO.succeed(onSuccessFnc(result)) - } - - } - protected def mapToSingleSuccessResponse[A]( effect: IO[ErrorResponse, A] ): IO[ErrorResponse, SingleSuccessResponse[A]] = { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala index 2b896873b..dadf0d513 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala @@ -16,97 +16,18 @@ package za.co.absa.atum.server.api.database -import cats.Show -import cats.data.NonEmptyList + import doobie.postgres.implicits._ import doobie.{Get, Put} -import io.circe.{Json => CirceJson} -import org.postgresql.jdbc.PgArray -import org.postgresql.util.PGobject -import io.circe.parser._ - -import scala.util.Try object DoobieImplicits { - private implicit val showPgArray: Show[PgArray] = Show.fromToString - implicit val getMapWithOptionStringValues: Get[Map[String, Option[String]]] = Get[Map[String, String]] .tmap(map => map.map { case (k, v) => k -> Option(v) }) - private def circeJsonListToPGJsonArrayString(jsonList: List[CirceJson]): String = { - val arrayElements = jsonList.map { x => - // Convert to compact JSON string and escape inner quotes - val escapedJsonString = x.noSpaces.replace("\"", "\\\"") - // Wrap in double quotes for the array element - s""""$escapedJsonString"""" - } - - arrayElements.mkString("{", ",", "}") - } - - private def pgArrayToListOfCirceJson(pgArray: PgArray): Either[String, List[CirceJson]] = { - Try { - Option(pgArray.getArray) match { - case Some(array: Array[_]) => array.collect { - case str: String => parse(str).toTry.get - case other => parse(other.toString).toTry.get - }.toList - case None => List.empty[CirceJson] - case _ => throw new IllegalArgumentException("Unexpected type encountered.") - } - } - .toEither - .left.map(_.getMessage) - } - object Sequence { - implicit val get: Get[Seq[String]] = Get[List[String]].map(_.toSeq) implicit val put: Put[Seq[String]] = Put[List[String]].contramap(_.toList) - - } - - object Json { - - implicit val jsonArrayPut: Put[List[CirceJson]] = { - Put.Advanced - .other[PGobject]( - NonEmptyList.of("json[]") - ) - .tcontramap { a => - val o = new PGobject - o.setType("json[]") - o.setValue(circeJsonListToPGJsonArrayString(a)) - o - } - } - - implicit val jsonArrayGet: Get[List[CirceJson]] = { - Get.Advanced - .other[PgArray]( - NonEmptyList.of("json[]") - ) - .temap(pgArray => pgArrayToListOfCirceJson(pgArray)) - } - - } - - object Jsonb { - - implicit val jsonbArrayPut: Put[List[CirceJson]] = { - Put.Advanced - .other[PGobject]( - NonEmptyList.of("jsonb[]") - ) - .tcontramap { a => - val o = new PGobject - o.setType("jsonb[]") - o.setValue(circeJsonListToPGJsonArrayString(a)) - o - } - } - } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 4cfac309f..1ad7c6392 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -25,11 +25,9 @@ import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWithAggStatus import zio._ -import zio.interop.catz._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ -import doobie.postgres.circe.jsonb.implicits.jsonbPut -import doobie.postgres.circe.json.implicits.jsonGet +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.{jsonbGet, jsonbPut} import io.circe.syntax.EncoderOps import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index 948232fd6..f98559444 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -26,11 +26,10 @@ import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithSt import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import zio._ -import zio.interop.catz._ import io.circe.syntax._ import doobie.postgres.implicits._ -import doobie.postgres.circe.jsonb.implicits.jsonbPut +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut class CreateOrUpdateAdditionalData(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[AdditionalDataSubmitDTO, Unit, Task]( diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 2a384ad0e..503830ffa 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -20,16 +20,15 @@ import doobie.implicits.toSqlInterpolator import za.co.absa.atum.model.dto.PartitioningSubmitDTO import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.db.fadb.DBSchema -import za.co.absa.db.fadb.doobie.{DoobieEngine, StatusWithData} +import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ -import zio.interop.catz._ import io.circe.syntax._ -import doobie.postgres.circe.jsonb.implicits.jsonbPut +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task]( diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala index 098fad56c..a0eeeb71d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala @@ -24,11 +24,10 @@ import za.co.absa.atum.server.model.{AdditionalDataFromDB, PartitioningForDB} import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWithAggStatus import za.co.absa.db.fadb.doobie.DoobieEngine -import zio.interop.catz.asyncInstance import zio.{Task, URLayer, ZIO, ZLayer} import io.circe.syntax._ import za.co.absa.atum.server.api.database.DoobieImplicits.getMapWithOptionStringValues -import doobie.postgres.circe.jsonb.implicits.jsonbPut +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 51c296c90..f8787322e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -25,12 +25,10 @@ import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWithAggStatus import zio._ -import zio.interop.catz._ -import io.circe.syntax._ +import io.circe.syntax.EncoderOps import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ -import doobie.postgres.circe.jsonb.implicits.jsonbPut -import doobie.postgres.circe.json.implicits.jsonGet +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.{jsonbGet, jsonbPut} import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 0398c96b5..a07e78588 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -26,13 +26,11 @@ import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ -import zio.interop.catz._ import io.circe.syntax._ import za.co.absa.atum.model.dto.MeasureResultDTO._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get -import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbArrayPut -import doobie.postgres.circe.jsonb.implicits.jsonbGet -import doobie.postgres.circe.jsonb.implicits.jsonbPut +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut import doobie.postgres.implicits._ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index a7ca1cb0b..bea89a0ec 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -21,33 +21,22 @@ import za.co.absa.db.fadb.exceptions.StatusException import za.co.absa.db.fadb.status.{FailedOrRow, FailedOrRows} import zio._ -// R - dbSingleResultCall[R] => IO[DatabaseError, R] -// Seq[R] dbMultipleResultCall => IO[DatabaseError, Seq[R]] -// FailedOrRow[R] ~ Either[StatusException, Row[R]] - dbSingleResultCallWithStatus => IO[DatabaseError, R] -// Seq[FailedOrRow[R]] ~ Seq[Either[StatusException, Row[R]]] - dbMultipleResultCallWithStatus => IO[DatabaseError, Seq[R]] -// FailedOrRows[R] ~ Either[StatusException, Seq[Row[R]]] - dbMultipleResultCallWithAggregatedStatus => IO[DatabaseError, Seq[R]] - -sealed trait PaginatedResult[R] -case class ResultHasMore[R](data: Seq[R]) extends PaginatedResult[R] -case class ResultNoMore[R](data: Seq[R]) extends PaginatedResult[R] - trait BaseRepository { - private def logOperationResult[R](operationName: String, dbFuncCall: Task[R]): - ZIO[Any, Throwable, R] = { + private def logOperationResult[R](operationName: String, dbFuncCall: Task[R]): ZIO[Any, Throwable, R] = { dbFuncCall .tap { - case Left(statusException: StatusException) => - ZIO.logError( - s"Exception caused by operation: '$operationName': " + - s"(${statusException.status.statusCode}), ${statusException.status.statusText}" - ) - case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") - } + case Left(statusException: StatusException) => + ZIO.logError( + s"Exception caused by operation: '$operationName': " + + s"(${statusException.status.statusCode}), ${statusException.status.statusText}" + ) + case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") + case _ => ZIO.logError(s"Operation '$operationName' did not return an Either") + } } - private def defaultErrorHandler[R](operationName: String): - PartialFunction[Throwable, DatabaseError] = { + private def defaultErrorHandler(operationName: String): PartialFunction[Throwable, DatabaseError] = { case statusException: StatusException => DatabaseError( s"Exception caused by operation: '$operationName': " + @@ -57,26 +46,7 @@ trait BaseRepository { DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") } - private def dbCall[R](dbFuncCall: Task[R], operationName: String): - IO[DatabaseError, R] = { - dbFuncCall - .zipLeft(ZIO.logDebug(s"Operation '$operationName' succeeded in database")) - .mapError(error => DatabaseError(error.getMessage)) - .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) - } - - def dbSingleResultCall[R](dbFuncCall: Task[R], operationName: String): - IO[DatabaseError, R] = { - dbCall(dbFuncCall, operationName) - } - - def dbMultipleResultCall[R](dbFuncCall: Task[Seq[R]], operationName: String): - IO[DatabaseError, Seq[R]] = { - dbCall(dbFuncCall, operationName) - } - - def dbSingleResultCallWithStatus[R](dbFuncCall: Task[FailedOrRow[R]], operationName: String): - IO[DatabaseError, R] = { + def dbSingleResultCallWithStatus[R](dbFuncCall: Task[FailedOrRow[R]], operationName: String): IO[DatabaseError, R] = { logOperationResult(operationName, dbFuncCall) .flatMap { case Left(statusException) => ZIO.fail(statusException) @@ -88,8 +58,7 @@ trait BaseRepository { .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } - def dbMultipleResultCallWithAggregatedStatus[R](dbFuncCall: Task[FailedOrRows[R]], operationName: String): - IO[DatabaseError, Seq[R]] = { + def dbMultipleResultCallWithAggregatedStatus[R](dbFuncCall: Task[FailedOrRows[R]], operationName: String): IO[DatabaseError, Seq[R]] = { logOperationResult(operationName, dbFuncCall) .flatMap { case Left(statusException) => ZIO.fail(statusException) @@ -101,23 +70,4 @@ trait BaseRepository { .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } - def dbPaginatedCall[R](dbFuncCall: Task[FailedOrRows[R]], operationName: String): - IO[DatabaseError, PaginatedResult[R]] = { - logOperationResult(operationName, dbFuncCall) - .flatMap { - case Left(statusException) => ZIO.fail(statusException) - case Right(value) => ZIO.succeed( - if (value.nonEmpty && value.head.functionStatus.statusCode == 11){ - ResultHasMore(value.map(_.data)) - } else { - ResultNoMore(value.map(_.data)) - } - ) - } - .mapError { - defaultErrorHandler(operationName) - } - .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) - } - } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepository.scala index f00e644c5..6f56aa145 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepository.scala @@ -24,6 +24,5 @@ import zio.macros.accessible @accessible trait FlowRepository { - def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): - IO[DatabaseError, Seq[CheckpointFromDB]] + def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala index 0276c744b..71152b335 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/FlowRepositoryImpl.scala @@ -25,8 +25,7 @@ import zio.interop.catz.asyncInstance class FlowRepositoryImpl(getFlowCheckpointsFn: GetFlowCheckpoints) extends FlowRepository with BaseRepository { - override def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): - IO[DatabaseError, Seq[CheckpointFromDB]] = { + override def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] = { dbMultipleResultCallWithAggregatedStatus(getFlowCheckpointsFn(checkpointQueryDTO), "getFlowCheckpoints") } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala index 49bd7cd66..936b25d0b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala @@ -24,23 +24,13 @@ import zio.macros.accessible @accessible trait PartitioningRepository { - def createPartitioningIfNotExists( - partitioningSubmitDTO: PartitioningSubmitDTO - ): IO[DatabaseError, Unit] + def createPartitioningIfNotExists(partitioningSubmitDTO: PartitioningSubmitDTO): IO[DatabaseError, Unit] - def getPartitioningMeasures( - partitioning: PartitioningDTO - ): IO[DatabaseError, Seq[MeasureDTO]] + def getPartitioningMeasures(partitioning: PartitioningDTO): IO[DatabaseError, Seq[MeasureDTO]] - def getPartitioningAdditionalData( - partitioning: PartitioningDTO - ): IO[DatabaseError, AdditionalDataDTO] + def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[DatabaseError, AdditionalDataDTO] - def createOrUpdateAdditionalData( - additionalData: AdditionalDataSubmitDTO - ): IO[DatabaseError, Unit] + def createOrUpdateAdditionalData(additionalData: AdditionalDataSubmitDTO): IO[DatabaseError, Unit] - def getPartitioningCheckpoints( - checkpointQueryDTO: CheckpointQueryDTO - ): IO[DatabaseError, Seq[CheckpointFromDB]] + def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index a7a0e9627..c6a49a4c0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -42,21 +42,15 @@ class PartitioningRepositoryImpl( getPartitioningCheckpointsFn: GetPartitioningCheckpoints ) extends PartitioningRepository with BaseRepository { - override def createPartitioningIfNotExists( - partitioningSubmitDTO: PartitioningSubmitDTO - ): IO[DatabaseError, Unit] = { + override def createPartitioningIfNotExists(partitioningSubmitDTO: PartitioningSubmitDTO): IO[DatabaseError, Unit] = { dbSingleResultCallWithStatus(createPartitioningIfNotExistsFn(partitioningSubmitDTO), "createPartitioningIfNotExists") } - override def createOrUpdateAdditionalData( - additionalData: AdditionalDataSubmitDTO - ): IO[DatabaseError, Unit] = { + override def createOrUpdateAdditionalData(additionalData: AdditionalDataSubmitDTO): IO[DatabaseError, Unit] = { dbSingleResultCallWithStatus(createOrUpdateAdditionalDataFn(additionalData), "createOrUpdateAdditionalData") } - override def getPartitioningMeasures( - partitioning: PartitioningDTO - ): IO[DatabaseError, Seq[MeasureDTO]] = { + override def getPartitioningMeasures(partitioning: PartitioningDTO): IO[DatabaseError, Seq[MeasureDTO]] = { dbMultipleResultCallWithAggregatedStatus(getPartitioningMeasuresFn(partitioning), "getPartitioningMeasures") } @@ -67,8 +61,7 @@ class PartitioningRepositoryImpl( ).map( _.map { case AdditionalDataFromDB(adName, adValue) => adName.get -> adValue }.toMap) } - override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): - IO[DatabaseError, Seq[CheckpointFromDB]] = { + override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] = { dbMultipleResultCallWithAggregatedStatus(getPartitioningCheckpointsFn(checkpointQueryDTO), "getPartitioningCheckpoints") } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala index 66633cf78..09a970857 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala @@ -22,20 +22,15 @@ import zio._ trait BaseService { - def repositoryCall[R]( - repositoryCall: IO[DatabaseError, R], - operationName: String - ): IO[ServiceError, R] = { + def repositoryCall[R](repositoryCall: IO[DatabaseError, R], operationName: String): IO[ServiceError, R] = { repositoryCall .mapError { case DatabaseError(message) => ServiceError(s"Failed to perform '$operationName': $message") } } - def repositoryCallWithStatus[R]( - repositoryCall: IO[DatabaseError, Either[StatusException, R]], - operationName: String - ): IO[ServiceError, Either[StatusException, R]] = { + def repositoryCallWithStatus[R](repositoryCall: IO[DatabaseError, Either[StatusException, R]], operationName: String + ): IO[ServiceError, Either[StatusException, R]] = { repositoryCall .mapError { case DatabaseError(message) => ServiceError(s"Failed to perform '$operationName': $message") diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowService.scala index 1464aed78..72945d705 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowService.scala @@ -23,7 +23,5 @@ import zio.macros.accessible @accessible trait FlowService { - def getFlowCheckpoints( - checkpointQueryDTO: CheckpointQueryDTO - ): IO[ServiceError, Seq[CheckpointDTO]] + def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[ServiceError, Seq[CheckpointDTO]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala index 4a87ecd9d..d050afb18 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala @@ -23,18 +23,13 @@ import zio.macros.accessible @accessible trait PartitioningService { - def createPartitioningIfNotExists(partitioningSubmitDTO: PartitioningSubmitDTO): - IO[ServiceError, Unit] + def createPartitioningIfNotExists(partitioningSubmitDTO: PartitioningSubmitDTO): IO[ServiceError, Unit] - def getPartitioningMeasures(partitioning: PartitioningDTO): - IO[ServiceError, Seq[MeasureDTO]] + def getPartitioningMeasures(partitioning: PartitioningDTO): IO[ServiceError, Seq[MeasureDTO]] - def getPartitioningAdditionalData(partitioning: PartitioningDTO): - IO[ServiceError, AdditionalDataDTO] + def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[ServiceError, AdditionalDataDTO] - def createOrUpdateAdditionalData(additionalData: AdditionalDataSubmitDTO): - IO[ServiceError, Unit] + def createOrUpdateAdditionalData(additionalData: AdditionalDataSubmitDTO): IO[ServiceError, Unit] - def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): - IO[ServiceError, Seq[CheckpointDTO]] + def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[ServiceError, Seq[CheckpointDTO]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala index b1470fee9..0ef3c5510 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala @@ -32,31 +32,25 @@ class PartitioningServiceImpl(partitioningRepository: PartitioningRepository) ) } - override def createOrUpdateAdditionalData( - additionalData: AdditionalDataSubmitDTO - ): IO[ServiceError, Unit] = { + override def createOrUpdateAdditionalData(additionalData: AdditionalDataSubmitDTO): IO[ServiceError, Unit] = { repositoryCall( partitioningRepository.createOrUpdateAdditionalData(additionalData), "createOrUpdateAdditionalData" ) } - override def getPartitioningMeasures(partitioning: PartitioningDTO): - IO[ServiceError, Seq[MeasureDTO]] = { + override def getPartitioningMeasures(partitioning: PartitioningDTO): IO[ServiceError, Seq[MeasureDTO]] = { repositoryCall( partitioningRepository.getPartitioningMeasures(partitioning), "getPartitioningMeasures" ) } - override def getPartitioningAdditionalData(partitioning: PartitioningDTO): - IO[ServiceError, AdditionalDataDTO] = { + override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[ServiceError, AdditionalDataDTO] = { repositoryCall( partitioningRepository.getPartitioningAdditionalData(partitioning), "getPartitioningAdditionalData" ) } - override def getPartitioningCheckpoints( - checkpointQueryDTO: CheckpointQueryDTO - ): IO[ServiceError, Seq[CheckpointDTO]] = { + override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[ServiceError, Seq[CheckpointDTO]] = { for { checkpointsFromDB <- repositoryCall( partitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO), "getPartitioningCheckpoints" diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala new file mode 100644 index 000000000..d1eee5934 --- /dev/null +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala @@ -0,0 +1,42 @@ +package za.co.absa.atum.server.api.database.flows.functions + +import za.co.absa.atum.server.ConfigProviderTest +import za.co.absa.atum.model.dto.{CheckpointQueryDTO, PartitionDTO, PartitioningDTO} +import za.co.absa.atum.server.api.TestTransactorProvider +import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import zio.interop.catz.asyncInstance +import zio.test.Assertion.failsWithA +import zio.{Scope, ZIO} +import zio.test._ + +object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { + + override def spec: Spec[TestEnvironment with Scope, Any] = { + + val partitioningDTO1: PartitioningDTO = Seq( + PartitionDTO("stringA", "stringA"), + PartitionDTO("stringB", "stringB") + ) + + suite("GetFlowCheckpointsIntegrationTests")( + test("Returns expected sequence of flow of Checkpoints with existing partitioning") { + val partitioningQueryDTO: CheckpointQueryDTO = CheckpointQueryDTO( + partitioning = partitioningDTO1, + limit = Some(10), + checkpointName = Some("checkpointName") + ) + + for { + getFlowCheckpoints <- ZIO.service[GetFlowCheckpoints] + exit <- getFlowCheckpoints(partitioningQueryDTO).exit + } yield assert(exit)(failsWithA[doobie.util.invariant.NonNullableColumnRead]) + } + ).provide( + GetFlowCheckpoints.layer, + PostgresDatabaseProvider.layer, + TestTransactorProvider.layerWithRollback + ) + } + +} + From f02bf72fbfdc0cc14d2da323509680588105c072 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 1 Aug 2024 15:20:41 +0200 Subject: [PATCH 28/31] Re-addressing GitHub comments --- project/Dependencies.scala | 2 - .../functions/GetPartitioningMeasures.scala | 6 +-- .../api/repository/BaseRepository.scala | 15 ++++--- .../repository/PartitioningRepository.scala | 9 +++- .../PartitioningRepositoryImpl.scala | 42 +++++++++++++------ .../atum/server/api/service/BaseService.scala | 9 ---- .../atum/server/model/CheckpointFromDB.scala | 32 +++++++------- .../atum/server/model/MeasureFromDB.scala | 22 ++++++++++ .../za/co/absa/atum/server/api/TestData.scala | 42 +++++++++++-------- .../GetFlowCheckpointsIntegrationTests.scala | 23 ++++++++-- ...titioningCheckpointsIntegrationTests.scala | 7 ++-- ...PartitioningMeasuresIntegrationTests.scala | 9 ++-- .../WriteCheckpointIntegrationTests.scala | 3 +- .../PartitioningRepositoryUnitTests.scala | 4 +- 14 files changed, 146 insertions(+), 79 deletions(-) create mode 100644 server/src/main/scala/za/co/absa/atum/server/model/MeasureFromDB.scala diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 687226999..6ca200e35 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -147,7 +147,6 @@ object Dependencies { // Fa-db lazy val faDbDoobie = faDbOrg %% "doobie" % Versions.fadb - lazy val pgCirceDoobie = "org.tpolecat" %% "doobie-postgres-circe" % "1.0.0-RC2" // aws lazy val awsSecretsManagerSdk = awsSdkOrg % "secretsmanager" % Versions.awssdk @@ -160,7 +159,6 @@ object Dependencies { Seq( faDbDoobie, - pgCirceDoobie, zioCore, zioMacros, zioLogging, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala index 96d4581db..65862f5e1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.implicits.toSqlInterpolator -import za.co.absa.atum.model.dto.{MeasureDTO, PartitioningDTO} +import za.co.absa.atum.model.dto.PartitioningDTO import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine @@ -25,15 +25,15 @@ import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWith import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ -import zio.interop.catz._ import io.circe.syntax._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.circe.jsonb.implicits.jsonbPut +import za.co.absa.atum.server.model.MeasureFromDB import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling class GetPartitioningMeasures (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, MeasureDTO, Task]( + extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, MeasureFromDB, Task]( values => Seq(fr"${PartitioningForDB.fromSeqPartitionDTO(values).asJson}")) with StandardStatusHandling with ByFirstErrorStatusAggregator { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index bea89a0ec..9f9764f91 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -23,7 +23,10 @@ import zio._ trait BaseRepository { - private def logOperationResult[R](operationName: String, dbFuncCall: Task[R]): ZIO[Any, Throwable, R] = { + private def logAndReturn[R]( + operationName: String, + dbFuncCall: Task[Either[StatusException, R]] + ): ZIO[Any, Throwable, Either[StatusException, R]] = { dbFuncCall .tap { case Left(statusException: StatusException) => @@ -32,7 +35,6 @@ trait BaseRepository { s"(${statusException.status.statusCode}), ${statusException.status.statusText}" ) case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") - case _ => ZIO.logError(s"Operation '$operationName' did not return an Either") } } @@ -47,7 +49,7 @@ trait BaseRepository { } def dbSingleResultCallWithStatus[R](dbFuncCall: Task[FailedOrRow[R]], operationName: String): IO[DatabaseError, R] = { - logOperationResult(operationName, dbFuncCall) + logAndReturn(operationName, dbFuncCall) .flatMap { case Left(statusException) => ZIO.fail(statusException) case Right(value) => ZIO.succeed(value.data) @@ -58,8 +60,11 @@ trait BaseRepository { .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } - def dbMultipleResultCallWithAggregatedStatus[R](dbFuncCall: Task[FailedOrRows[R]], operationName: String): IO[DatabaseError, Seq[R]] = { - logOperationResult(operationName, dbFuncCall) + def dbMultipleResultCallWithAggregatedStatus[R]( + dbFuncCall: Task[FailedOrRows[R]], + operationName: String + ): IO[DatabaseError, Seq[R]] = { + logAndReturn(operationName, dbFuncCall) .flatMap { case Left(statusException) => ZIO.fail(statusException) case Right(value) => ZIO.succeed(value.map(_.data)) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala index 936b25d0b..b34bda040 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepository.scala @@ -16,7 +16,14 @@ package za.co.absa.atum.server.api.repository -import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} +import za.co.absa.atum.model.dto.{ + AdditionalDataDTO, + AdditionalDataSubmitDTO, + CheckpointQueryDTO, + MeasureDTO, + PartitioningDTO, + PartitioningSubmitDTO +} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB import zio.IO diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index c6a49a4c0..fd9fe8cb0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -19,31 +19,39 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.{ AdditionalDataDTO, AdditionalDataSubmitDTO, - CheckpointQueryDTO, MeasureDTO, - PartitioningDTO, PartitioningSubmitDTO} + CheckpointQueryDTO, + MeasureDTO, + PartitioningDTO, + PartitioningSubmitDTO +} +import za.co.absa.atum.server.model.MeasureFromDB import za.co.absa.atum.server.api.database.runs.functions.{ CreateOrUpdateAdditionalData, CreatePartitioningIfNotExists, GetPartitioningAdditionalData, GetPartitioningCheckpoints, - GetPartitioningMeasures} + GetPartitioningMeasures +} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB import zio._ import zio.interop.catz.asyncInstance import za.co.absa.atum.server.model.AdditionalDataFromDB - class PartitioningRepositoryImpl( createPartitioningIfNotExistsFn: CreatePartitioningIfNotExists, getPartitioningMeasuresFn: GetPartitioningMeasures, getPartitioningAdditionalDataFn: GetPartitioningAdditionalData, createOrUpdateAdditionalDataFn: CreateOrUpdateAdditionalData, getPartitioningCheckpointsFn: GetPartitioningCheckpoints -) extends PartitioningRepository with BaseRepository { +) extends PartitioningRepository + with BaseRepository { override def createPartitioningIfNotExists(partitioningSubmitDTO: PartitioningSubmitDTO): IO[DatabaseError, Unit] = { - dbSingleResultCallWithStatus(createPartitioningIfNotExistsFn(partitioningSubmitDTO), "createPartitioningIfNotExists") + dbSingleResultCallWithStatus( + createPartitioningIfNotExistsFn(partitioningSubmitDTO), + "createPartitioningIfNotExists" + ) } override def createOrUpdateAdditionalData(additionalData: AdditionalDataSubmitDTO): IO[DatabaseError, Unit] = { @@ -52,18 +60,25 @@ class PartitioningRepositoryImpl( override def getPartitioningMeasures(partitioning: PartitioningDTO): IO[DatabaseError, Seq[MeasureDTO]] = { dbMultipleResultCallWithAggregatedStatus(getPartitioningMeasuresFn(partitioning), "getPartitioningMeasures") + .map(_.map { case MeasureFromDB(measureName, measuredColumns) => + MeasureDTO(measureName.get, measuredColumns.get) + }) } override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[DatabaseError, AdditionalDataDTO] = { dbMultipleResultCallWithAggregatedStatus( - getPartitioningAdditionalDataFn(partitioning) - , "getPartitioningAdditionalData" - ).map( _.map { case AdditionalDataFromDB(adName, adValue) => adName.get -> adValue }.toMap) + getPartitioningAdditionalDataFn(partitioning), + "getPartitioningAdditionalData" + ).map(_.map { case AdditionalDataFromDB(adName, adValue) => adName.get -> adValue }.toMap) } - override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[DatabaseError, Seq[CheckpointFromDB]] = { - dbMultipleResultCallWithAggregatedStatus(getPartitioningCheckpointsFn(checkpointQueryDTO), - "getPartitioningCheckpoints") + override def getPartitioningCheckpoints( + checkpointQueryDTO: CheckpointQueryDTO + ): IO[DatabaseError, Seq[CheckpointFromDB]] = { + dbMultipleResultCallWithAggregatedStatus( + getPartitioningCheckpointsFn(checkpointQueryDTO), + "getPartitioningCheckpoints" + ) } } @@ -88,6 +103,7 @@ object PartitioningRepositoryImpl { getPartitioningMeasures, getPartitioningAdditionalData, createOrUpdateAdditionalData, - getPartitioningCheckpoints) + getPartitioningCheckpoints + ) } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala index 09a970857..680dcc22d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala @@ -17,7 +17,6 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.server.api.exception.{DatabaseError, ServiceError} -import za.co.absa.db.fadb.exceptions.StatusException import zio._ trait BaseService { @@ -29,12 +28,4 @@ trait BaseService { } } - def repositoryCallWithStatus[R](repositoryCall: IO[DatabaseError, Either[StatusException, R]], operationName: String - ): IO[ServiceError, Either[StatusException, R]] = { - repositoryCall - .mapError { case DatabaseError(message) => - ServiceError(s"Failed to perform '$operationName': $message") - } - } - } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 705e6c319..3da84e7fe 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -23,14 +23,14 @@ import java.time.ZonedDateTime import java.util.UUID case class CheckpointFromDB( - idCheckpoint: UUID, - checkpointName: String, - author: String, - measuredByAtumAgent: Boolean = false, - measureName: String, - measuredColumns: Seq[String], - measurementValue: Json, // it's easier to convert this attribute to our `MeasurementDTO` after we received this as JSON from DB - checkpointStartTime: ZonedDateTime, + idCheckpoint: Option[UUID], + checkpointName: Option[String], + author: Option[String], + measuredByAtumAgent: Option[Boolean], + measureName: Option[String], + measuredColumns: Option[Seq[String]], + measurementValue: Option[Json], // it's easier to convert this attribute to our `MeasurementDTO` after we received this as JSON from DB + checkpointStartTime: Option[ZonedDateTime], checkpointEndTime: Option[ZonedDateTime] ) @@ -40,25 +40,25 @@ object CheckpointFromDB { partitioning: PartitioningDTO, checkpointQueryResult: CheckpointFromDB ): Either[DecodingFailure, CheckpointDTO] = { - val measureResultOrErr = checkpointQueryResult.measurementValue.as[MeasureResultDTO] + val measureResultOrErr = checkpointQueryResult.measurementValue.get.as[MeasureResultDTO] measureResultOrErr match { case Left(err) => Left(err) case Right(measureResult) => Right( CheckpointDTO( - id = checkpointQueryResult.idCheckpoint, - name = checkpointQueryResult.checkpointName, - author = checkpointQueryResult.author, - measuredByAtumAgent = checkpointQueryResult.measuredByAtumAgent, + id = checkpointQueryResult.idCheckpoint.get, + name = checkpointQueryResult.checkpointName.get, + author = checkpointQueryResult.author.get, + measuredByAtumAgent = checkpointQueryResult.measuredByAtumAgent.get, partitioning = partitioning, - processStartTime = checkpointQueryResult.checkpointStartTime, + processStartTime = checkpointQueryResult.checkpointStartTime.get, processEndTime = checkpointQueryResult.checkpointEndTime, measurements = Set( MeasurementDTO( measure = MeasureDTO( - measureName = checkpointQueryResult.measureName, - measuredColumns = checkpointQueryResult.measuredColumns + measureName = checkpointQueryResult.measureName.get, + measuredColumns = checkpointQueryResult.measuredColumns.get ), result = measureResult ) diff --git a/server/src/main/scala/za/co/absa/atum/server/model/MeasureFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/MeasureFromDB.scala new file mode 100644 index 000000000..f1250c844 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/model/MeasureFromDB.scala @@ -0,0 +1,22 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.model + +case class MeasureFromDB ( + measureName: Option[String], + measuredColumns: Option[Seq[String]] +) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala index 19acbfdc9..caf3143ea 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala @@ -24,6 +24,7 @@ import java.time.ZonedDateTime import java.util.UUID import MeasureResultDTO.TypedValue import za.co.absa.atum.model.ResultValueType +import za.co.absa.atum.server.model.MeasureFromDB trait TestData { @@ -58,6 +59,10 @@ trait TestData { protected val measureDTO1: MeasureDTO = MeasureDTO("count1", Seq("col_A1", "col_B1")) protected val measureDTO2: MeasureDTO = MeasureDTO("count2", Seq("col_A2", "col_B2")) + // Measure from DB + protected val measureFromDB1: MeasureFromDB = MeasureFromDB(Some("count1"), Some(Seq("col_A1", "col_B1"))) + protected val measureFromDB2: MeasureFromDB = MeasureFromDB(Some("count2"), Some(Seq("col_A2", "col_B2"))) + // Additional Data protected val additionalDataDTO1: AdditionalDataDTO = Map( "key1" -> Some("value1"), @@ -189,13 +194,13 @@ trait TestData { // Checkpoint From DB protected val checkpointFromDB1: CheckpointFromDB = CheckpointFromDB( - idCheckpoint = checkpointDTO1.id, - checkpointName = checkpointQueryDTO1.checkpointName.get, - author = "author", - measuredByAtumAgent = true, - measureName = measureDTO1.measureName, - measuredColumns = measureDTO1.measuredColumns.toIndexedSeq, - measurementValue = parser + idCheckpoint = Some(checkpointDTO1.id), + checkpointName = checkpointQueryDTO1.checkpointName, + author = Some("author"), + measuredByAtumAgent = Some(true), + measureName = Some(measureDTO1.measureName), + measuredColumns = Some(measureDTO1.measuredColumns.toIndexedSeq), + measurementValue = Some(parser .parse( """ |{ @@ -218,25 +223,28 @@ trait TestData { ) .getOrElse { throw new Exception("Failed to parse JSON") - }, - checkpointStartTime = checkpointDTO1.processStartTime, + }), + checkpointStartTime = Some(checkpointDTO1.processStartTime), checkpointEndTime = checkpointDTO1.processEndTime ) protected val checkpointFromDB2: CheckpointFromDB = checkpointFromDB1 .copy( - idCheckpoint = checkpointDTO2.id, - checkpointName = checkpointQueryDTO2.checkpointName.get, - author = "author2", - measuredByAtumAgent = true, - measureName = measureDTO2.measureName, - measuredColumns = measureDTO2.measuredColumns.toIndexedSeq, - checkpointStartTime = checkpointDTO2.processStartTime, + idCheckpoint = Some(checkpointDTO2.id), + checkpointName = checkpointQueryDTO2.checkpointName, + author = Some("author2"), + measuredByAtumAgent = Some(true), + measureName = Some(measureDTO2.measureName), + measuredColumns = Some(measureDTO2.measuredColumns.toIndexedSeq), + checkpointStartTime = Some(checkpointDTO2.processStartTime), checkpointEndTime = checkpointDTO2.processEndTime ) protected val checkpointFromDB3: CheckpointFromDB = checkpointFromDB1 - .copy(idCheckpoint = checkpointDTO3.id, checkpointStartTime = checkpointDTO3.processStartTime) + .copy( + idCheckpoint = Some(checkpointDTO3.id), + checkpointStartTime = Some(checkpointDTO3.processStartTime) + ) protected def createAtumContextDTO(partitioningSubmitDTO: PartitioningSubmitDTO): AtumContextDTO = { val measures: Set[MeasureDTO] = Set(MeasureDTO("count", Seq("*"))) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala index d1eee5934..a9cc2a8b3 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala @@ -1,11 +1,28 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.server.api.database.flows.functions import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.model.dto.{CheckpointQueryDTO, PartitionDTO, PartitioningDTO} import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import za.co.absa.db.fadb.exceptions.DataNotFoundException +import za.co.absa.db.fadb.status.FunctionStatus import zio.interop.catz.asyncInstance -import zio.test.Assertion.failsWithA import zio.{Scope, ZIO} import zio.test._ @@ -28,8 +45,8 @@ object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { for { getFlowCheckpoints <- ZIO.service[GetFlowCheckpoints] - exit <- getFlowCheckpoints(partitioningQueryDTO).exit - } yield assert(exit)(failsWithA[doobie.util.invariant.NonNullableColumnRead]) + result <- getFlowCheckpoints(partitioningQueryDTO) + } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) } ).provide( GetFlowCheckpoints.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala index b7da127af..79444d510 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala @@ -20,8 +20,9 @@ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.model.dto.{CheckpointQueryDTO, PartitionDTO, PartitioningDTO} import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import za.co.absa.db.fadb.exceptions.DataNotFoundException +import za.co.absa.db.fadb.status.FunctionStatus import zio.interop.catz.asyncInstance -import zio.test.Assertion.failsWithA import zio.{Scope, ZIO} import zio.test._ @@ -44,8 +45,8 @@ object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { for { getPartitioningCheckpoints <- ZIO.service[GetPartitioningCheckpoints] - exit <- getPartitioningCheckpoints(partitioningQueryDTO).exit - } yield assert(exit)(failsWithA[doobie.util.invariant.NonNullableColumnRead]) + result <- getPartitioningCheckpoints(partitioningQueryDTO) + } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) } ).provide( GetPartitioningCheckpoints.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala index 1ae885596..cfb291dbb 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala @@ -20,9 +20,10 @@ import za.co.absa.atum.model.dto.{PartitionDTO, PartitioningDTO} import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import za.co.absa.db.fadb.exceptions.DataNotFoundException import zio.interop.catz.asyncInstance -import zio.test.Assertion.failsWithA -import zio.test.{Spec, TestEnvironment, assert} +import za.co.absa.db.fadb.status.FunctionStatus +import zio.test.{Spec, TestEnvironment, assertTrue} import zio.{Scope, ZIO} object GetPartitioningMeasuresIntegrationTests extends ConfigProviderTest { @@ -34,8 +35,8 @@ object GetPartitioningMeasuresIntegrationTests extends ConfigProviderTest { val partitioningDTO: PartitioningDTO = Seq(PartitionDTO("string11", "string11"), PartitionDTO("string12", "string12")) for { getPartitioningMeasures <- ZIO.service[GetPartitioningMeasures] - result <- getPartitioningMeasures(partitioningDTO).exit - } yield assert(result)(failsWithA[doobie.util.invariant.NonNullableColumnRead]) + result <- getPartitioningMeasures(partitioningDTO) + } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) } ).provide( GetPartitioningMeasures.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index c2e423f37..2ced19af0 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -31,7 +31,8 @@ import zio.test._ import java.time.ZonedDateTime import java.util.UUID -object WriteCheckpointIntegrationTests extends ConfigProviderTest { +object +WriteCheckpointIntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala index 9992b1248..59ff2095b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala @@ -60,8 +60,8 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { when(getPartitioningMeasuresMock.apply(partitioningDTO1)) .thenReturn(ZIO.right(Seq( - Row(FunctionStatus(0, "success"), measureDTO1), - Row(FunctionStatus(0, "success"), measureDTO2))) + Row(FunctionStatus(0, "success"), measureFromDB1), + Row(FunctionStatus(0, "success"), measureFromDB2))) ) when(getPartitioningMeasuresMock.apply(partitioningDTO2)).thenReturn(ZIO.fail(DatabaseError("boom!"))) From c96e90a6f707f4f2e90dedb320cc1fd212a56b2e Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 1 Aug 2024 15:37:27 +0200 Subject: [PATCH 29/31] Addind Todo comment --- model/src/main/scala/za/co/absa/atum/model/dto/package.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala index 2d3952241..8618e9cb0 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala @@ -23,6 +23,7 @@ package object dto { type PartitioningDTO = Seq[PartitionDTO] type AdditionalDataDTO = Map[String, Option[String]] + // Todo. This implicit definition should not be defined here, so it is to be addressed in PR#221 // Implicit encoders and decoders for AdditionalDataDTO implicit val decodeAdditionalDataDTO: Decoder[AdditionalDataDTO] = Decoder.decodeMap[String, Option[String]] implicit val encodeAdditionalDataDTO: Encoder[AdditionalDataDTO] = Encoder.encodeMap[String, Option[String]] From 30054048ab6d291067ebcd970612eeeac2a424e0 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 2 Aug 2024 09:43:59 +0200 Subject: [PATCH 30/31] Fixing format comments --- model/src/main/scala/za/co/absa/atum/model/dto/package.scala | 2 +- .../runs/functions/WriteCheckpointIntegrationTests.scala | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala index 8618e9cb0..b025d2b27 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala @@ -23,7 +23,7 @@ package object dto { type PartitioningDTO = Seq[PartitionDTO] type AdditionalDataDTO = Map[String, Option[String]] - // Todo. This implicit definition should not be defined here, so it is to be addressed in PR#221 + // Todo. This implicit definition should not be defined here, so it is to be addressed in Ticket #221 // Implicit encoders and decoders for AdditionalDataDTO implicit val decodeAdditionalDataDTO: Decoder[AdditionalDataDTO] = Decoder.decodeMap[String, Option[String]] implicit val encodeAdditionalDataDTO: Encoder[AdditionalDataDTO] = Encoder.encodeMap[String, Option[String]] diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index 2ced19af0..c2e423f37 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -31,8 +31,7 @@ import zio.test._ import java.time.ZonedDateTime import java.util.UUID -object -WriteCheckpointIntegrationTests extends ConfigProviderTest { +object WriteCheckpointIntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { From 94750cef9ce19a780374a4b50da69b4354bd0795 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Sun, 4 Aug 2024 07:49:35 +0200 Subject: [PATCH 31/31] Fixing format comments Implemented suggestions --- server/README.md | 17 ++++-- server/prometheus.yml | 4 +- .../scala/za/co/absa/atum/server/Main.scala | 2 +- .../api/controller/BaseController.scala | 4 +- .../api/controller/FlowController.scala | 4 +- .../server/api/database/DoobieImplicits.scala | 1 - .../api/database/TransactorProvider.scala | 6 +- .../flows/functions/GetFlowCheckpoints.scala | 13 ++-- .../CreateOrUpdateAdditionalData.scala | 13 ++-- .../CreatePartitioningIfNotExists.scala | 13 ++-- .../GetPartitioningAdditionalData.scala | 13 ++-- .../GetPartitioningCheckpoints.scala | 15 ++--- .../functions/GetPartitioningMeasures.scala | 14 ++--- .../runs/functions/WriteCheckpoint.scala | 7 ++- .../atum/server/api/http/BaseEndpoints.scala | 7 ++- .../absa/atum/server/api/http/Endpoints.scala | 20 ++++--- .../co/absa/atum/server/api/http/Routes.scala | 9 ++- .../api/service/CheckpointServiceImpl.scala | 6 +- .../server/api/service/FlowServiceImpl.scala | 15 +++-- .../api/service/PartitioningService.scala | 10 +++- .../api/service/PartitioningServiceImpl.scala | 33 ++++++---- .../server/model/AdditionalDataFromDB.scala | 2 +- .../atum/server/model/CheckpointFromDB.scala | 20 ++++--- .../atum/server/model/ErrorResponse.scala | 16 ++--- .../atum/server/model/MeasureFromDB.scala | 2 +- .../atum/server/model/SuccessResponse.scala | 6 +- server/src/test/resources/logback-test.xml | 4 +- .../absa/atum/server/ConfigProviderTest.scala | 1 - .../za/co/absa/atum/server/api/TestData.scala | 21 +++---- .../CheckpointControllerUnitTests.scala | 8 ++- .../controller/FlowControllerUnitTests.scala | 4 +- .../PartitioningControllerUnitTests.scala | 1 - .../GetFlowCheckpointsIntegrationTests.scala | 1 - ...UpdateAdditionalDataIntegrationTests.scala | 2 +- ...ioningAdditionalDataIntegrationTests.scala | 5 +- ...titioningCheckpointsIntegrationTests.scala | 1 - ...PartitioningMeasuresIntegrationTests.scala | 5 +- .../WriteCheckpointIntegrationTests.scala | 5 +- .../api/http/BaseEndpointsUnitTests.scala | 60 +++++++++---------- .../CheckpointRepositoryUnitTests.scala | 4 +- .../repository/FlowRepositoryUnitTests.scala | 13 ++-- .../PartitioningRepositoryUnitTests.scala | 26 ++++---- .../service/CheckpointServiceUnitTests.scala | 3 +- .../api/service/FlowServiceUnitTests.scala | 7 +-- .../PartitioningServiceUnitTests.scala | 14 +++-- .../aws/AwsSecretsProviderUnitTests.scala | 2 +- 46 files changed, 255 insertions(+), 204 deletions(-) diff --git a/server/README.md b/server/README.md index 5dd693703..35be75d25 100644 --- a/server/README.md +++ b/server/README.md @@ -9,21 +9,24 @@ To create a jar file that can be executed: > java -jar server/target/jvm-2.13/*.jar ``` -If you want to quickly build and run from sbt you can run using the command below (alternatively you can execute za.co.absa.atum.server.Main within your IDE). This deploys it to `localhost:8080`. +If you want to quickly build and run from sbt you can run using the command below (alternatively you can execute +za.co.absa.atum.server.Main within your IDE). This deploys it to `localhost:8080`. ```shell sbt "server/runMain za.co.absa.atum.server.Main" ``` -### REST API Reference +### REST API Reference The REST API exposes a Swagger Documentation UI which documents all the HTTP endpoints exposed. It can be found at **{REST_API_HOST}/docs/** (e.g. `http://localhost:8080/docs/`) ### Observability metrics -Optionally you can run server with monitoring that collects metrics about http communication and/or jvm/zio runtime. `intervalInSeconds` parameter refers to frequency of data collection from its runtime environment. -Monitoring of http communication is based on intercepting of http calls therefore `intervalInSeconds` parameter does not apply. +Optionally you can run server with monitoring that collects metrics about http communication and/or jvm/zio +runtime. `intervalInSeconds` parameter refers to frequency of data collection from its runtime environment. +Monitoring of http communication is based on intercepting of http calls therefore `intervalInSeconds` parameter does not +apply. ``` { @@ -41,5 +44,7 @@ Monitoring of http communication is based on intercepting of http calls therefor } ``` -When monitoring enabled, the application exposes `http://localhost:8080/metrics` and/or `http://localhost:8080/zio-metrics` endpoints which can be scraped by Prometheus. -For testing purposes there is [docker-compose.yml](./docker-compose.yml) file which can be used to start up dockerized Prometheus and Grafana instances. Prometheus scraping configs are defined in [prometheus.yml](./prometheus.yml) file. +When monitoring enabled, the application exposes `http://localhost:8080/metrics` +and/or `http://localhost:8080/zio-metrics` endpoints which can be scraped by Prometheus. +For testing purposes there is [docker-compose.yml](./docker-compose.yml) file which can be used to start up dockerized +Prometheus and Grafana instances. Prometheus scraping configs are defined in [prometheus.yml](./prometheus.yml) file. diff --git a/server/prometheus.yml b/server/prometheus.yml index 5abcfa42c..7d0a27cad 100644 --- a/server/prometheus.yml +++ b/server/prometheus.yml @@ -19,14 +19,14 @@ scrape_configs: - job_name: 'atum_server_http4s' metrics_path: /metrics static_configs: - - targets: ['host.docker.internal:8080'] + - targets: [ 'host.docker.internal:8080' ] labels: env: 'local' app: 'atum' - job_name: 'atum_server_zio_runtime' metrics_path: /zio-metrics static_configs: - - targets: ['host.docker.internal:8080'] + - targets: [ 'host.docker.internal:8080' ] labels: env: 'local' app: 'atum' diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index 34295349e..19ab07fb8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -67,7 +67,7 @@ object Main extends ZIOAppDefault with Server { // enabling conditionally collection of ZIO runtime metrics and default JVM metrics if (jvmMonitoringConfig.enabled) { ZLayer.succeed(MetricsConfig(Duration.ofSeconds(jvmMonitoringConfig.intervalInSeconds))) ++ - Runtime.enableRuntimeMetrics.unit ++ DefaultJvmMetrics.live.unit + Runtime.enableRuntimeMetrics.unit ++ DefaultJvmMetrics.live.unit } else { ZLayer.succeed(MetricsConfig(Duration.ofSeconds(Long.MaxValue))) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index a4aeac9dc..88c133487 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -32,8 +32,8 @@ trait BaseController { .mapError { serviceError: ServiceError => InternalServerErrorResponse(serviceError.message) } - .flatMap { - result => ZIO.succeed(onSuccessFnc(result)) + .flatMap { result => + ZIO.succeed(onSuccessFnc(result)) } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowController.scala index d122a93c2..684c8e8ac 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowController.scala @@ -24,5 +24,7 @@ import zio.macros.accessible @accessible trait FlowController { - def getFlowCheckpointsV2(checkpointQueryDTO: CheckpointQueryDTO): IO[ErrorResponse, MultiSuccessResponse[CheckpointDTO]] + def getFlowCheckpointsV2( + checkpointQueryDTO: CheckpointQueryDTO + ): IO[ErrorResponse, MultiSuccessResponse[CheckpointDTO]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala index dadf0d513..a37c3629c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.server.api.database - import doobie.postgres.implicits._ import doobie.{Get, Put} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/TransactorProvider.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/TransactorProvider.scala index a82a07aa5..2ce29fd26 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/TransactorProvider.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/TransactorProvider.scala @@ -32,10 +32,12 @@ object TransactorProvider { awsConfig <- ZIO.config[AwsConfig](AwsConfig.config) awsSecretsProvider <- ZIO.service[AwsSecretsProvider] - password <- awsSecretsProvider.getSecretValue(awsConfig.dbPasswordSecretName) + password <- awsSecretsProvider + .getSecretValue(awsConfig.dbPasswordSecretName) // fallback to password property's value from postgres section of reference.conf; useful for local testing .orElse { - ZIO.logError("Credentials were not retrieved from AWS, falling back to config value.") + ZIO + .logError("Credentials were not retrieved from AWS, falling back to config value.") .as(postgresConfig.password) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 1ad7c6392..7d0c8e079 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -33,16 +33,17 @@ import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusA import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunctionWithAggStatus[CheckpointQueryDTO, CheckpointFromDB, Task]( - values => Seq( + extends DoobieMultipleResultFunctionWithAggStatus[CheckpointQueryDTO, CheckpointFromDB, Task](values => + Seq( fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", fr"${values.limit}", fr"${values.checkpointName}" ) - ) with StandardStatusHandling with ByFirstErrorStatusAggregator { - override val fieldsToSelect: Seq[String] = Seq( - "status", - "status_text", + ) + with StandardStatusHandling + with ByFirstErrorStatusAggregator { + + override def fieldsToSelect: Seq[String] = super.fieldsToSelect ++ Seq( "id_checkpoint", "checkpoint_name", "author", diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index f98559444..ca8281a55 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -32,13 +32,14 @@ import doobie.postgres.implicits._ import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut class CreateOrUpdateAdditionalData(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[AdditionalDataSubmitDTO, Unit, Task]( - values => Seq( - fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", - fr"${values.additionalData.map{ case (k, v) => (k, v.orNull)}}", - fr"${values.author}" + extends DoobieSingleResultFunctionWithStatus[AdditionalDataSubmitDTO, Unit, Task](values => + Seq( + fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", + fr"${values.additionalData.map { case (k, v) => (k, v.orNull) }}", + fr"${values.author}" + ) ) - ) with StandardStatusHandling + with StandardStatusHandling object CreateOrUpdateAdditionalData { val layer: URLayer[PostgresDatabaseProvider, CreateOrUpdateAdditionalData] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 503830ffa..a9f1e8b13 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -31,13 +31,14 @@ import io.circe.syntax._ import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task]( - values => Seq( - fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", - fr"${values.authorIfNew}", - fr"${values.parentPartitioning.map(PartitioningForDB.fromSeqPartitionDTO).map(_.asJson)}" + extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task](values => + Seq( + fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", + fr"${values.authorIfNew}", + fr"${values.parentPartitioning.map(PartitioningForDB.fromSeqPartitionDTO).map(_.asJson)}" + ) ) - ) with StandardStatusHandling + with StandardStatusHandling object CreatePartitioningIfNotExists { val layer: URLayer[PostgresDatabaseProvider, CreatePartitioningIfNotExists] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala index a0eeeb71d..968e1081b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala @@ -31,11 +31,14 @@ import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling -class GetPartitioningAdditionalData (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, AdditionalDataFromDB, Task]( - values => Seq(fr"${PartitioningForDB.fromSeqPartitionDTO(values).asJson}")) - with StandardStatusHandling with ByFirstErrorStatusAggregator { - override val fieldsToSelect: Seq[String] = Seq("status", "status_text", "ad_name", "ad_value") +class GetPartitioningAdditionalData(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) + extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, AdditionalDataFromDB, Task](values => + Seq(fr"${PartitioningForDB.fromSeqPartitionDTO(values).asJson}") + ) + with StandardStatusHandling + with ByFirstErrorStatusAggregator { + + override def fieldsToSelect: Seq[String] = super.fieldsToSelect ++ Seq("ad_name", "ad_value") } object GetPartitioningAdditionalData { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index f8787322e..4901b867a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -32,17 +32,18 @@ import za.co.absa.db.fadb.doobie.postgres.circe.implicits.{jsonbGet, jsonbPut} import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling -class GetPartitioningCheckpoints (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunctionWithAggStatus[CheckpointQueryDTO, CheckpointFromDB, Task] ( - values => Seq( +class GetPartitioningCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) + extends DoobieMultipleResultFunctionWithAggStatus[CheckpointQueryDTO, CheckpointFromDB, Task](values => + Seq( fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", fr"${values.limit}", fr"${values.checkpointName}" ) - ) with StandardStatusHandling with ByFirstErrorStatusAggregator { - override val fieldsToSelect: Seq[String] = Seq( - "status", - "status_text", + ) + with StandardStatusHandling + with ByFirstErrorStatusAggregator { + + override def fieldsToSelect: Seq[String] = super.fieldsToSelect ++ Seq( "id_checkpoint", "checkpoint_name", "author", diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala index 65862f5e1..9b653365d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala @@ -32,13 +32,14 @@ import za.co.absa.atum.server.model.MeasureFromDB import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling -class GetPartitioningMeasures (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, MeasureFromDB, Task]( - values => Seq(fr"${PartitioningForDB.fromSeqPartitionDTO(values).asJson}")) - with StandardStatusHandling with ByFirstErrorStatusAggregator { - - override val fieldsToSelect: Seq[String] = Seq("status", "status_text", "measure_name", "measured_columns") +class GetPartitioningMeasures(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) + extends DoobieMultipleResultFunctionWithAggStatus[PartitioningDTO, MeasureFromDB, Task](values => + Seq(fr"${PartitioningForDB.fromSeqPartitionDTO(values).asJson}") + ) + with StandardStatusHandling + with ByFirstErrorStatusAggregator { + override def fieldsToSelect: Seq[String] = super.fieldsToSelect ++ Seq("measure_name", "measured_columns") } object GetPartitioningMeasures { @@ -48,4 +49,3 @@ object GetPartitioningMeasures { } yield new GetPartitioningMeasures()(Runs, dbProvider.dbEngine) } } - diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index a07e78588..ce6c5c129 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -34,8 +34,8 @@ import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut import doobie.postgres.implicits._ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task]( - values => Seq( + extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task](values => + Seq( fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", fr"${values.id}", fr"${values.name}", @@ -44,7 +44,8 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) fr"${values.measurements.toList.map(_.asJson)}", fr"${values.measuredByAtumAgent}", fr"${values.author}" - )) + ) + ) with StandardStatusHandling object WriteCheckpoint { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala index 422f044ef..3ef8edfb1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala @@ -19,7 +19,12 @@ package za.co.absa.atum.server.api.http import sttp.model.StatusCode import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.json.circe.jsonBody -import za.co.absa.atum.server.model.{BadRequestResponse, ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{ + BadRequestResponse, + ErrorResponse, + GeneralErrorResponse, + InternalServerErrorResponse +} import sttp.tapir.typelevel.MatchType import sttp.tapir.ztapir._ import sttp.tapir.{EndpointOutput, PublicEndpoint} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index f86103204..792d68a25 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.server.api.http - import sttp.model.StatusCode import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.ztapir._ @@ -27,7 +26,6 @@ import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import sttp.tapir.{PublicEndpoint, endpoint} - trait Endpoints extends BaseEndpoints { protected val createCheckpointEndpointV1: PublicEndpoint[CheckpointDTO, ErrorResponse, CheckpointDTO, Any] = { @@ -38,7 +36,8 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[CheckpointDTO]) } - protected val createCheckpointEndpointV2: PublicEndpoint[CheckpointDTO, ErrorResponse, SingleSuccessResponse[CheckpointDTO], Any] = { + protected val createCheckpointEndpointV2 + : PublicEndpoint[CheckpointDTO, ErrorResponse, SingleSuccessResponse[CheckpointDTO], Any] = { apiV2.post .in(CreateCheckpoint) .in(jsonBody[CheckpointDTO]) @@ -46,7 +45,8 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[SingleSuccessResponse[CheckpointDTO]]) } - protected val createPartitioningEndpointV1: PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, AtumContextDTO, Any] = { + protected val createPartitioningEndpointV1 + : PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, AtumContextDTO, Any] = { apiV1.post .in(pathToAPIv1CompatibleFormat(CreatePartitioning)) .in(jsonBody[PartitioningSubmitDTO]) @@ -54,7 +54,8 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[AtumContextDTO]) } - protected val createPartitioningEndpointV2: PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, SingleSuccessResponse[AtumContextDTO], Any] = { + protected val createPartitioningEndpointV2 + : PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, SingleSuccessResponse[AtumContextDTO], Any] = { apiV2.post .in(CreatePartitioning) .in(jsonBody[PartitioningSubmitDTO]) @@ -62,7 +63,8 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[SingleSuccessResponse[AtumContextDTO]]) } - protected val createOrUpdateAdditionalDataEndpointV2: PublicEndpoint[AdditionalDataSubmitDTO, ErrorResponse, SingleSuccessResponse[AdditionalDataSubmitDTO], Any] = { + protected val createOrUpdateAdditionalDataEndpointV2 + : PublicEndpoint[AdditionalDataSubmitDTO, ErrorResponse, SingleSuccessResponse[AdditionalDataSubmitDTO], Any] = { apiV2.post .in(CreateOrUpdateAdditionalData) .in(jsonBody[AdditionalDataSubmitDTO]) @@ -70,7 +72,8 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[SingleSuccessResponse[AdditionalDataSubmitDTO]]) } - protected val getPartitioningCheckpointsEndpointV2: PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { + protected val getPartitioningCheckpointsEndpointV2 + : PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { apiV2.get .in(GetPartitioningCheckpoints) .in(jsonBody[CheckpointQueryDTO]) @@ -78,7 +81,8 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[MultiSuccessResponse[CheckpointDTO]]) } - protected val getFlowCheckpointsEndpointV2: PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { + protected val getFlowCheckpointsEndpointV2 + : PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { apiV2.post .in(GetFlowCheckpoints) .in(jsonBody[CheckpointQueryDTO]) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala index 8f03690c1..1da88efce 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala @@ -42,10 +42,13 @@ trait Routes extends Endpoints with ServerOptions { createServerEndpoint(createCheckpointEndpointV2, CheckpointController.createCheckpointV2), createServerEndpoint(createPartitioningEndpointV1, PartitioningController.createPartitioningIfNotExistsV1), createServerEndpoint(createPartitioningEndpointV2, PartitioningController.createPartitioningIfNotExistsV2), - createServerEndpoint(createOrUpdateAdditionalDataEndpointV2, PartitioningController.createOrUpdateAdditionalDataV2), + createServerEndpoint( + createOrUpdateAdditionalDataEndpointV2, + PartitioningController.createOrUpdateAdditionalDataV2 + ), createServerEndpoint(getPartitioningCheckpointsEndpointV2, PartitioningController.getPartitioningCheckpointsV2), createServerEndpoint(getFlowCheckpointsEndpointV2, FlowController.getFlowCheckpointsV2), - createServerEndpoint(healthEndpoint, (_: Unit) => ZIO.unit), + createServerEndpoint(healthEndpoint, (_: Unit) => ZIO.unit) ) ZHttp4sServerInterpreter[HttpEnv.Env](http4sServerOptions(metricsInterceptorOption)).from(endpoints).toRoutes } @@ -61,7 +64,7 @@ trait Routes extends Endpoints with ServerOptions { createPartitioningEndpointV2, createOrUpdateAdditionalDataEndpointV2, getPartitioningCheckpointsEndpointV2, - getFlowCheckpointsEndpointV2, + getFlowCheckpointsEndpointV2 ) ZHttp4sServerInterpreter[HttpEnv.Env](http4sServerOptions(None)) .from(SwaggerInterpreter().fromEndpoints[HttpEnv.F](endpoints, SwaggerApiName, SwaggerApiVersion)) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala index 0616e8adf..aae123ea4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala @@ -21,12 +21,12 @@ import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.repository.CheckpointRepository import zio._ -class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) - extends CheckpointService with BaseService { +class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends CheckpointService with BaseService { override def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] = { repositoryCall( - checkpointRepository.writeCheckpoint(checkpointDTO), "saveCheckpoint" + checkpointRepository.writeCheckpoint(checkpointDTO), + "saveCheckpoint" ) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala index 93e333c67..5f788b86c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala @@ -22,19 +22,18 @@ import za.co.absa.atum.server.api.repository.FlowRepository import za.co.absa.atum.server.model.CheckpointFromDB import zio._ - -class FlowServiceImpl(flowRepository: FlowRepository) - extends FlowService with BaseService { +class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with BaseService { override def getFlowCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[ServiceError, Seq[CheckpointDTO]] = { for { checkpointsFromDB <- repositoryCall( - flowRepository.getFlowCheckpoints(checkpointQueryDTO), "getFlowCheckpoints" + flowRepository.getFlowCheckpoints(checkpointQueryDTO), + "getFlowCheckpoints" ) - checkpointDTOs <- ZIO.foreach(checkpointsFromDB) { - checkpointFromDB => - ZIO.fromEither(CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, checkpointFromDB)) - .mapError(error => ServiceError(error.getMessage)) + checkpointDTOs <- ZIO.foreach(checkpointsFromDB) { checkpointFromDB => + ZIO + .fromEither(CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, checkpointFromDB)) + .mapError(error => ServiceError(error.getMessage)) } } yield checkpointDTOs } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala index d050afb18..2f5d06a20 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningService.scala @@ -16,7 +16,15 @@ package za.co.absa.atum.server.api.service -import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataSubmitDTO, CheckpointDTO, CheckpointQueryDTO, MeasureDTO, PartitioningDTO, PartitioningSubmitDTO} +import za.co.absa.atum.model.dto.{ + AdditionalDataDTO, + AdditionalDataSubmitDTO, + CheckpointDTO, + CheckpointQueryDTO, + MeasureDTO, + PartitioningDTO, + PartitioningSubmitDTO +} import za.co.absa.atum.server.api.exception.ServiceError import zio.IO import zio.macros.accessible diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala index 0ef3c5510..f764d0249 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala @@ -23,42 +23,49 @@ import za.co.absa.atum.server.model.CheckpointFromDB import zio._ class PartitioningServiceImpl(partitioningRepository: PartitioningRepository) - extends PartitioningService with BaseService { + extends PartitioningService + with BaseService { - override def createPartitioningIfNotExists(partitioningSubmitDTO: PartitioningSubmitDTO): - IO[ServiceError, Unit] = { + override def createPartitioningIfNotExists(partitioningSubmitDTO: PartitioningSubmitDTO): IO[ServiceError, Unit] = { repositoryCall( - partitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO), "createPartitioningIfNotExists" + partitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO), + "createPartitioningIfNotExists" ) } override def createOrUpdateAdditionalData(additionalData: AdditionalDataSubmitDTO): IO[ServiceError, Unit] = { repositoryCall( - partitioningRepository.createOrUpdateAdditionalData(additionalData), "createOrUpdateAdditionalData" + partitioningRepository.createOrUpdateAdditionalData(additionalData), + "createOrUpdateAdditionalData" ) } override def getPartitioningMeasures(partitioning: PartitioningDTO): IO[ServiceError, Seq[MeasureDTO]] = { repositoryCall( - partitioningRepository.getPartitioningMeasures(partitioning), "getPartitioningMeasures" + partitioningRepository.getPartitioningMeasures(partitioning), + "getPartitioningMeasures" ) } override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[ServiceError, AdditionalDataDTO] = { repositoryCall( - partitioningRepository.getPartitioningAdditionalData(partitioning), "getPartitioningAdditionalData" + partitioningRepository.getPartitioningAdditionalData(partitioning), + "getPartitioningAdditionalData" ) } - override def getPartitioningCheckpoints(checkpointQueryDTO: CheckpointQueryDTO): IO[ServiceError, Seq[CheckpointDTO]] = { + override def getPartitioningCheckpoints( + checkpointQueryDTO: CheckpointQueryDTO + ): IO[ServiceError, Seq[CheckpointDTO]] = { for { checkpointsFromDB <- repositoryCall( - partitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO), "getPartitioningCheckpoints" + partitioningRepository.getPartitioningCheckpoints(checkpointQueryDTO), + "getPartitioningCheckpoints" ) - checkpointDTOs <- ZIO.foreach(checkpointsFromDB) { - checkpointFromDB => - ZIO.fromEither(CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, checkpointFromDB)) - .mapError(error => ServiceError(error.getMessage)) + checkpointDTOs <- ZIO.foreach(checkpointsFromDB) { checkpointFromDB => + ZIO + .fromEither(CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, checkpointFromDB)) + .mapError(error => ServiceError(error.getMessage)) } } yield checkpointDTOs diff --git a/server/src/main/scala/za/co/absa/atum/server/model/AdditionalDataFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/AdditionalDataFromDB.scala index c22c8a4aa..4816ab2bb 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/AdditionalDataFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/AdditionalDataFromDB.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.model -case class AdditionalDataFromDB ( +case class AdditionalDataFromDB( adName: Option[String], adValue: Option[String] ) diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 3da84e7fe..0eba1e01c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -23,15 +23,17 @@ import java.time.ZonedDateTime import java.util.UUID case class CheckpointFromDB( - idCheckpoint: Option[UUID], - checkpointName: Option[String], - author: Option[String], - measuredByAtumAgent: Option[Boolean], - measureName: Option[String], - measuredColumns: Option[Seq[String]], - measurementValue: Option[Json], // it's easier to convert this attribute to our `MeasurementDTO` after we received this as JSON from DB - checkpointStartTime: Option[ZonedDateTime], - checkpointEndTime: Option[ZonedDateTime] + idCheckpoint: Option[UUID], + checkpointName: Option[String], + author: Option[String], + measuredByAtumAgent: Option[Boolean], + measureName: Option[String], + measuredColumns: Option[Seq[String]], + measurementValue: Option[ + Json + ], // it's easier to convert this attribute to our `MeasurementDTO` after we received this as JSON from DB + checkpointStartTime: Option[ZonedDateTime], + checkpointEndTime: Option[ZonedDateTime] ) object CheckpointFromDB { diff --git a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala index bf17272b4..65d316931 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.server.model - import io.circe._ import io.circe.generic.semiauto._ @@ -27,28 +26,25 @@ object ErrorResponse { implicit val encodeErrorResponse: Encoder[ErrorResponse] = deriveEncoder } - sealed trait ErrorResponse extends ResponseEnvelope { - def message: String - } +sealed trait ErrorResponse extends ResponseEnvelope { + def message: String +} - final case class BadRequestResponse(message: String, requestId: UUID = UUID.randomUUID()) - extends ErrorResponse +final case class BadRequestResponse(message: String, requestId: UUID = UUID.randomUUID()) extends ErrorResponse object BadRequestResponse { implicit val decodeBadRequestResponse: Decoder[BadRequestResponse] = deriveDecoder implicit val encodeBadRequestResponse: Encoder[BadRequestResponse] = deriveEncoder } - final case class GeneralErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) - extends ErrorResponse +final case class GeneralErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) extends ErrorResponse object GeneralErrorResponse { implicit val decodeGeneralErrorResponse: Decoder[GeneralErrorResponse] = deriveDecoder implicit val encodeGeneralErrorResponse: Encoder[GeneralErrorResponse] = deriveEncoder } - final case class InternalServerErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) - extends ErrorResponse +final case class InternalServerErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) extends ErrorResponse object InternalServerErrorResponse { implicit val decodeInternalServerErrorResponse: Decoder[InternalServerErrorResponse] = deriveDecoder diff --git a/server/src/main/scala/za/co/absa/atum/server/model/MeasureFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/MeasureFromDB.scala index f1250c844..dd122c069 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/MeasureFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/MeasureFromDB.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.model -case class MeasureFromDB ( +case class MeasureFromDB( measureName: Option[String], measuredColumns: Option[Seq[String]] ) diff --git a/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala b/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala index f3898cc66..05d27f2f7 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala @@ -25,16 +25,14 @@ object SuccessResponse { sealed trait SuccessResponse extends ResponseEnvelope - case class SingleSuccessResponse[T](data: T, requestId: UUID = UUID.randomUUID()) - extends SuccessResponse + case class SingleSuccessResponse[T](data: T, requestId: UUID = UUID.randomUUID()) extends SuccessResponse object SingleSuccessResponse { implicit def encoder[T: Encoder]: Encoder[SingleSuccessResponse[T]] = deriveEncoder implicit def decoder[T: Decoder]: Decoder[SingleSuccessResponse[T]] = deriveDecoder } - case class MultiSuccessResponse[T](data: Seq[T], requestId: UUID = UUID.randomUUID()) - extends SuccessResponse + case class MultiSuccessResponse[T](data: Seq[T], requestId: UUID = UUID.randomUUID()) extends SuccessResponse object MultiSuccessResponse { implicit def encoder[T: Encoder]: Encoder[MultiSuccessResponse[T]] = deriveEncoder diff --git a/server/src/test/resources/logback-test.xml b/server/src/test/resources/logback-test.xml index 6e2afba1d..7b7cb3bb9 100644 --- a/server/src/test/resources/logback-test.xml +++ b/server/src/test/resources/logback-test.xml @@ -6,7 +6,7 @@ - + - + diff --git a/server/src/test/scala/za/co/absa/atum/server/ConfigProviderTest.scala b/server/src/test/scala/za/co/absa/atum/server/ConfigProviderTest.scala index bf57c9fda..eee346043 100644 --- a/server/src/test/scala/za/co/absa/atum/server/ConfigProviderTest.scala +++ b/server/src/test/scala/za/co/absa/atum/server/ConfigProviderTest.scala @@ -26,4 +26,3 @@ abstract class ConfigProviderTest extends ZIOSpec[Unit] { Runtime.setConfigProvider(TypesafeConfigProvider.fromResourcePath()) } - diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala index caf3143ea..b846e7973 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala @@ -152,13 +152,13 @@ trait TestData { protected val checkpointQueryDTO1: CheckpointQueryDTO = CheckpointQueryDTO( partitioning = partitioningDTO1, limit = Option(2), - checkpointName = Option("checkpointName"), + checkpointName = Option("checkpointName") ) protected val checkpointQueryDTO2: CheckpointQueryDTO = CheckpointQueryDTO( partitioning = partitioningDTO2, limit = Option(5), - checkpointName = Option("noCheckpoints"), + checkpointName = Option("noCheckpoints") ) protected val checkpointQueryDTO3: CheckpointQueryDTO = CheckpointQueryDTO( @@ -200,9 +200,10 @@ trait TestData { measuredByAtumAgent = Some(true), measureName = Some(measureDTO1.measureName), measuredColumns = Some(measureDTO1.measuredColumns.toIndexedSeq), - measurementValue = Some(parser - .parse( - """ + measurementValue = Some( + parser + .parse( + """ |{ | "mainValue": { | "value": "123", @@ -220,10 +221,11 @@ trait TestData { | } |} |""".stripMargin - ) - .getOrElse { - throw new Exception("Failed to parse JSON") - }), + ) + .getOrElse { + throw new Exception("Failed to parse JSON") + } + ), checkpointStartTime = Some(checkpointDTO1.processStartTime), checkpointEndTime = checkpointDTO1.processEndTime ) @@ -237,7 +239,6 @@ trait TestData { measuredColumns = Some(measureDTO2.measuredColumns.toIndexedSeq), checkpointStartTime = Some(checkpointDTO2.processStartTime), checkpointEndTime = checkpointDTO2.processEndTime - ) protected val checkpointFromDB3: CheckpointFromDB = checkpointFromDB1 diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index bc0450582..7554742eb 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -47,10 +47,14 @@ object CheckpointControllerUnitTests extends ZIOSpecDefault with TestData { } yield assertTrue(result == checkpointDTO1) }, test("Returns expected InternalServerErrorResponse") { - assertZIO(CheckpointController.createCheckpointV1(checkpointDTO3).exit)(failsWithA[InternalServerErrorResponse]) + assertZIO(CheckpointController.createCheckpointV1(checkpointDTO3).exit)( + failsWithA[InternalServerErrorResponse] + ) }, test("Returns expected GeneralErrorResponse") { - assertZIO(CheckpointController.createCheckpointV1(checkpointDTO2).exit)(failsWithA[InternalServerErrorResponse]) + assertZIO(CheckpointController.createCheckpointV1(checkpointDTO2).exit)( + failsWithA[InternalServerErrorResponse] + ) } ) ).provide( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala index be3f586b7..4cf152128 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala @@ -43,13 +43,11 @@ object FlowControllerUnitTests extends ZIOSpecDefault with TestData { failsWithA[InternalServerErrorResponse] ) }, - test("Returns expected CheckpointDTO") { for { result <- FlowController.getFlowCheckpointsV2(checkpointQueryDTO2) - } yield assertTrue (result.data == Seq(checkpointDTO2)) + } yield assertTrue(result.data == Seq(checkpointDTO2)) } - ) ).provide( FlowControllerImpl.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala index 333c79ea6..5a78b5012 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala @@ -83,7 +83,6 @@ object PartitioningControllerUnitTests extends ZIOSpecDefault with TestData { ) } ), - suite("GetPartitioningCheckpointsSuite")( test("Returns expected Seq[MeasureDTO]") { for { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala index a9cc2a8b3..67ad6c05a 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala @@ -56,4 +56,3 @@ object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { } } - diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalDataIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalDataIntegrationTests.scala index 344b6048b..94bb6564c 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalDataIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalDataIntegrationTests.scala @@ -34,7 +34,7 @@ object CreateOrUpdateAdditionalDataIntegrationTests extends ConfigProviderTest { test("Returns expected Right with Unit") { val additionalDataSubmitDTO = AdditionalDataSubmitDTO( partitioning = Seq(PartitionDTO("key1", "val1"), PartitionDTO("key2", "val2")), - additionalData = Map[String, Option[String]]( + additionalData = Map[String, Option[String]]( "ownership" -> Some("total"), "role" -> Some("primary") ), diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalDataIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalDataIntegrationTests.scala index abdd8eca3..3bced5c34 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalDataIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalDataIntegrationTests.scala @@ -30,7 +30,8 @@ object GetPartitioningAdditionalDataIntegrationTests extends ConfigProviderTest override def spec: Spec[TestEnvironment with Scope, Any] = { suite("GetPartitioningAdditionalDataSuite")( test("Returns expected sequence of Additional data with provided partitioning") { - val partitioningDTO: PartitioningDTO = Seq(PartitionDTO("stringA", "stringB"), PartitionDTO("string2", "string2")) + val partitioningDTO: PartitioningDTO = + Seq(PartitionDTO("stringA", "stringB"), PartitionDTO("string2", "string2")) for { getPartitioningAdditionalData <- ZIO.service[GetPartitioningAdditionalData] result <- getPartitioningAdditionalData(partitioningDTO) @@ -39,7 +40,7 @@ object GetPartitioningAdditionalDataIntegrationTests extends ConfigProviderTest ).provide( GetPartitioningAdditionalData.layer, PostgresDatabaseProvider.layer, - TestTransactorProvider.layerWithRollback, + TestTransactorProvider.layerWithRollback ) } } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala index 79444d510..ab38e39ca 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala @@ -56,4 +56,3 @@ object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { } } - diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala index cfb291dbb..c43bfb634 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasuresIntegrationTests.scala @@ -32,11 +32,12 @@ object GetPartitioningMeasuresIntegrationTests extends ConfigProviderTest { suite("GetPartitioningMeasuresSuite")( test("Returns expected sequence of Measures with existing partitioning") { - val partitioningDTO: PartitioningDTO = Seq(PartitionDTO("string11", "string11"), PartitionDTO("string12", "string12")) + val partitioningDTO: PartitioningDTO = + Seq(PartitionDTO("string11", "string11"), PartitionDTO("string12", "string12")) for { getPartitioningMeasures <- ZIO.service[GetPartitioningMeasures] result <- getPartitioningMeasures(partitioningDTO) - } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) + } yield assertTrue (result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) } ).provide( GetPartitioningMeasures.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index c2e423f37..2f1b8ffe5 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -45,8 +45,9 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { partitioning = Seq(PartitionDTO("key4", "value4")), processStartTime = ZonedDateTime.now(), processEndTime = Option(ZonedDateTime.now()), - measurements = - Set(MeasurementDTO(MeasureDTO("count", Seq("*")), MeasureResultDTO(TypedValue("1", ResultValueType.LongValue)))) + measurements = Set( + MeasurementDTO(MeasureDTO("count", Seq("*")), MeasureResultDTO(TypedValue("1", ResultValueType.LongValue))) + ) ) for { writeCheckpoint <- ZIO.service[WriteCheckpoint] diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/BaseEndpointsUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/BaseEndpointsUnitTests.scala index 37d54e8bc..e8798974b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/BaseEndpointsUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/BaseEndpointsUnitTests.scala @@ -32,54 +32,54 @@ class BaseEndpointsUnitTests extends AnyFlatSpec { "pathToAPIv1CompatibleFormat" should "successfully convert our standard API path format to format compatible with API V1 (kebab)" in { - val input = "create-checkpoint" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "createCheckpoint" - assert(actual == expected) - } + val input = "create-checkpoint" + val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) + val expected = "createCheckpoint" + assert(actual == expected) + } "pathToAPIv1CompatibleFormat" should "successfully convert our standard API path format to format compatible with API V1 (kebab2)" in { - val input = "create-check-point2" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "createCheckPoint2" - assert(actual == expected) - } + val input = "create-check-point2" + val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) + val expected = "createCheckPoint2" + assert(actual == expected) + } "pathToAPIv1CompatibleFormat" should "successfully convert our standard API path format to format compatible with API V1 (kebab3)" in { - val input = "Create-check-" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "createCheck" - assert(actual == expected) - } + val input = "Create-check-" + val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) + val expected = "createCheck" + assert(actual == expected) + } "pathToAPIv1CompatibleFormat" should "successfully convert our standard API path format to format compatible with API V1 (snake)" in { - val input = "_create_check_point" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "CreateCheckPoint" - assert(actual == expected) - } + val input = "_create_check_point" + val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) + val expected = "CreateCheckPoint" + assert(actual == expected) + } "pathToAPIv1CompatibleFormat" should "successfully convert our standard API path format to format compatible with API V1 (kebab and snake)" in { - val input = "Create-check_Point" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "createCheckPoint" - assert(actual == expected) - } + val input = "Create-check_Point" + val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) + val expected = "createCheckPoint" + assert(actual == expected) + } "pathToAPIv1CompatibleFormat" should "successfully convert our standard API path format to format compatible with API V1 (one word)" in { - val input = "createcheckpoint" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "createcheckpoint" - assert(actual == expected) - } + val input = "createcheckpoint" + val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) + val expected = "createcheckpoint" + assert(actual == expected) + } } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index 7017056ca..e7af6ecf1 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -52,7 +52,9 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Left with StatusException") { for { result <- CheckpointRepository.writeCheckpoint(checkpointDTO2).exit - } yield assertTrue(result == Exit.fail(DatabaseError("Operation 'writeCheckpoint' failed with unexpected error: null"))) + } yield assertTrue( + result == Exit.fail(DatabaseError("Operation 'writeCheckpoint' failed with unexpected error: null")) + ) }, test("Returns expected DatabaseError") { assertZIO(CheckpointRepository.writeCheckpoint(checkpointDTO3).exit)(failsWithA[DatabaseError]) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/FlowRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/FlowRepositoryUnitTests.scala index ce4e09f71..7f34bab93 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/FlowRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/FlowRepositoryUnitTests.scala @@ -32,9 +32,10 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { when(getFlowCheckpointsMock.apply(checkpointQueryDTO1)).thenReturn(ZIO.fail(new Exception("boom!"))) when(getFlowCheckpointsMock.apply(checkpointQueryDTO2)) - .thenReturn(ZIO.right(Seq( - Row(FunctionStatus(0, "success"),checkpointFromDB1), - Row(FunctionStatus(0, "success"), checkpointFromDB2))) + .thenReturn( + ZIO.right( + Seq(Row(FunctionStatus(0, "success"), checkpointFromDB1), Row(FunctionStatus(0, "success"), checkpointFromDB2)) + ) ) private val getFlowCheckpointsMockLayer = ZLayer.succeed(getFlowCheckpointsMock) @@ -52,11 +53,11 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { for { result <- FlowRepository.getFlowCheckpoints(checkpointQueryDTO2) } yield assertTrue(result == Seq(checkpointFromDB1, checkpointFromDB2)) - }, - ), + } + ) ).provide( FlowRepositoryImpl.layer, - getFlowCheckpointsMockLayer, + getFlowCheckpointsMockLayer ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala index 59ff2095b..1c054613f 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala @@ -28,7 +28,6 @@ import zio.test.Assertion.failsWithA import zio.test._ import za.co.absa.atum.server.model.AdditionalDataFromDB - object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { // Create Partitioning Mocks @@ -59,9 +58,10 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { private val getPartitioningMeasuresMock = mock(classOf[GetPartitioningMeasures]) when(getPartitioningMeasuresMock.apply(partitioningDTO1)) - .thenReturn(ZIO.right(Seq( - Row(FunctionStatus(0, "success"), measureFromDB1), - Row(FunctionStatus(0, "success"), measureFromDB2))) + .thenReturn( + ZIO.right( + Seq(Row(FunctionStatus(0, "success"), measureFromDB1), Row(FunctionStatus(0, "success"), measureFromDB2)) + ) ) when(getPartitioningMeasuresMock.apply(partitioningDTO2)).thenReturn(ZIO.fail(DatabaseError("boom!"))) @@ -98,7 +98,13 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Left with StatusException") { for { result <- PartitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO2).exit - } yield assertTrue(result == Exit.fail(DatabaseError("Exception caused by operation: 'createPartitioningIfNotExists': (50) error in Partitioning data"))) + } yield assertTrue( + result == Exit.fail( + DatabaseError( + "Exception caused by operation: 'createPartitioningIfNotExists': (50) error in Partitioning data" + ) + ) + ) }, test("Returns expected DatabaseError") { assertZIO(PartitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO3).exit)( @@ -106,7 +112,6 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { ) } ), - suite("CreateOrUpdateAdditionalDataSuite")( test("Returns expected Right with Unit") { for { @@ -116,7 +121,11 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Left with StatusException") { for { result <- PartitioningRepository.createOrUpdateAdditionalData(additionalDataSubmitDTO2).exit - } yield assertTrue(result == Exit.fail(DatabaseError("Exception caused by operation: 'createOrUpdateAdditionalData': (50) error in AD data"))) + } yield assertTrue( + result == Exit.fail( + DatabaseError("Exception caused by operation: 'createOrUpdateAdditionalData': (50) error in AD data") + ) + ) }, test("Returns expected DatabaseError") { assertZIO(PartitioningRepository.createOrUpdateAdditionalData(additionalDataSubmitDTO3).exit)( @@ -124,7 +133,6 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { ) } ), - suite("GetPartitioningMeasuresSuite")( test("Returns expected Seq") { for { @@ -137,7 +145,6 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { ) } ), - suite("GetPartitioningAdditionalDataSuite")( test("Returns expected Right with Map") { for { @@ -150,7 +157,6 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { ) } ), - suite("GetPartitioningCheckpointsSuite")( test("Returns expected Seq") { for { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala index 63ad67413..f73c21c8c 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala @@ -29,8 +29,7 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { private val checkpointRepositoryMock = mock(classOf[CheckpointRepository]) when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO1)).thenReturn(ZIO.succeed(())) - when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO2)). - thenReturn(ZIO.fail(DatabaseError("error in data"))) + when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO2)).thenReturn(ZIO.fail(DatabaseError("error in data"))) when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO3)).thenReturn(ZIO.fail(DatabaseError("boom!"))) private val checkpointRepositoryMockLayer = ZLayer.succeed(checkpointRepositoryMock) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/FlowServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/FlowServiceUnitTests.scala index 8b70529ae..da1bfba87 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/FlowServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/FlowServiceUnitTests.scala @@ -45,12 +45,11 @@ object FlowServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Seq[CheckpointDTO]") { for { result <- FlowService.getFlowCheckpoints(checkpointQueryDTO2) - } yield assertTrue{ + } yield assertTrue { result == Seq(checkpointDTO2) } - }, - - ), + } + ) ).provide( FlowServiceImpl.layer, flowRepositoryMockLayer diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala index 8dd6a22b4..34da663c0 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala @@ -69,7 +69,9 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Left with StatusException") { for { result <- PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO2).exit - } yield assertTrue(result == Exit.fail(ServiceError("Failed to perform 'createPartitioningIfNotExists': error in data"))) + } yield assertTrue( + result == Exit.fail(ServiceError("Failed to perform 'createPartitioningIfNotExists': error in data")) + ) }, test("Returns expected ServiceError") { assertZIO(PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO3).exit)( @@ -86,7 +88,9 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Left with StatusException") { for { result <- PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO2).exit - } yield assertTrue(result == Exit.fail(ServiceError("Failed to perform 'createOrUpdateAdditionalData': error in AD data"))) + } yield assertTrue( + result == Exit.fail(ServiceError("Failed to perform 'createOrUpdateAdditionalData': error in AD data")) + ) }, test("Returns expected ServiceError") { assertZIO(PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO3).exit)( @@ -98,7 +102,7 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Seq[MeasureDTO]") { for { result <- PartitioningService.getPartitioningMeasures(partitioningDTO1) - } yield assertTrue{ + } yield assertTrue { result == Seq(measureDTO1, measureDTO2) } }, @@ -112,7 +116,7 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Seq[AdditionalDataDTO]") { for { result <- PartitioningService.getPartitioningAdditionalData(partitioningDTO1) - } yield assertTrue{result == additionalDataDTO1} + } yield assertTrue { result == additionalDataDTO1 } }, test("Returns expected ServiceError") { assertZIO(PartitioningService.getPartitioningAdditionalData(partitioningDTO2).exit)( @@ -124,7 +128,7 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Right with Seq[CheckpointDTO]") { for { result <- PartitioningService.getPartitioningCheckpoints(checkpointQueryDTO1) - } yield assertTrue{ + } yield assertTrue { result == Seq(checkpointDTO1, checkpointDTO2.copy(partitioning = checkpointDTO1.partitioning)) } }, diff --git a/server/src/test/scala/za/co/absa/atum/server/aws/AwsSecretsProviderUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/aws/AwsSecretsProviderUnitTests.scala index b53fdc2ef..49547e921 100644 --- a/server/src/test/scala/za/co/absa/atum/server/aws/AwsSecretsProviderUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/aws/AwsSecretsProviderUnitTests.scala @@ -39,7 +39,7 @@ object AwsSecretsProviderUnitTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { suite("AwsSecretsProviderSuite")( - test("GetSecretValue returns expected secret's value"){ + test("GetSecretValue returns expected secret's value") { for { awsConfig <- ZIO.config[AwsConfig](AwsConfig.config) awsSecretValue <- AwsSecretsProvider.getSecretValue(awsConfig.dbPasswordSecretName)