Skip to content

Commit 96ffa33

Browse files
committed
* some omitted files
1 parent 1488d1f commit 96ffa33

File tree

8 files changed

+609
-0
lines changed

8 files changed

+609
-0
lines changed
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
/*
2+
* Copyright 2021 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package za.co.absa.atum.reader.exceptions
18+
19+
class ReaderException(message: String) extends Exception(message)
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright 2021 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package za.co.absa.atum.reader.exceptions
18+
19+
import sttp.model.{StatusCode, Uri}
20+
import za.co.absa.atum.model.envelopes.ErrorResponse
21+
22+
abstract class RequestException(message: String) extends ReaderException(message)
23+
24+
25+
object RequestException {
26+
type CirceError = io.circe.Error
27+
28+
final case class HttpException(
29+
message: String,
30+
statusCode: StatusCode,
31+
errorResponse: ErrorResponse,
32+
request: Uri
33+
) extends RequestException(message)
34+
35+
final case class ParsingException(
36+
message: String,
37+
body: String
38+
) extends RequestException(message)
39+
object ParsingException {
40+
def fromCirceError(error: CirceError, body: String): ParsingException = {
41+
ParsingException(error.getMessage, body)
42+
}
43+
}
44+
45+
46+
final case class NoDataException(
47+
message: String
48+
) extends RequestException(message)
49+
}
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Copyright 2021 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package za.co.absa.atum.reader.result
18+
19+
import sttp.monad.MonadError
20+
21+
abstract class AbstractPage [T <: Iterable[_], F[_]: MonadError] {
22+
def items: T
23+
def hasNext: Boolean
24+
def limit: Int
25+
def pageStart: Long
26+
def pageEnd: Long
27+
28+
def pageSize: Int = (pageEnd - pageStart).toInt + 1
29+
def hasPrior: Boolean = pageStart > 0
30+
}
31+
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
* Copyright 2021 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package za.co.absa.atum.reader.result
18+
19+
import sttp.monad.MonadError
20+
import sttp.monad.syntax._
21+
import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestResult}
22+
import za.co.absa.atum.reader.exceptions.RequestException.NoDataException
23+
import za.co.absa.atum.reader.result.GroupedPage.GroupPageRoller
24+
25+
import scala.collection.immutable.ListMap
26+
27+
case class GroupedPage[K, V, F[_]: MonadError](
28+
items: ListMap[K, Vector[V]],
29+
hasNext: Boolean,
30+
limit: Int,
31+
pageStart: Long,
32+
pageEnd: Long,
33+
private[reader] val pageRoller: GroupPageRoller[K, V, F]
34+
) extends AbstractPage[Map[K, Vector[V]], F] {
35+
36+
def apply(key: K): Vector[V] = items(key)
37+
def keys: Iterable[K] = items.keys
38+
def groupCount: Int = items.size
39+
40+
def map[K1, V1](f: ((K, Vector[V])) => (K1, Vector[V1])): GroupedPage[K1, V1, F] = {
41+
val newItems = items.map(f)
42+
val newPageRoller: GroupPageRoller[K1, V1, F] = (limit, offset) => pageRoller(limit, offset).map(_.map(_.map(f)))
43+
this.copy(items = newItems, pageRoller = newPageRoller)
44+
}
45+
46+
def mapValues[B](f: V => B): GroupedPage[K, B, F] = {
47+
def mapper(item: (K, Vector[V])): (K, Vector[B]) = (item._1, item._2.map(f))
48+
49+
val newItems = items.map(mapper)
50+
val newPageRoller: GroupPageRoller[K, B, F] = (limit, offset) => pageRoller(limit, offset).map(_.map(_.mapValues(f)))
51+
this.copy(items = newItems, pageRoller = newPageRoller)
52+
53+
}
54+
55+
def prior(newPageSize: Int): F[RequestResult[GroupedPage[K, V, F]]] = {
56+
if (hasPrior) {
57+
val newOffset = (pageStart - limit).max(0)
58+
pageRoller(newPageSize, newOffset)
59+
} else {
60+
MonadError[F].unit(RequestFail(NoDataException("No prior page")))
61+
}
62+
}
63+
64+
def prior: F[RequestResult[GroupedPage[K, V, F]]] = prior(limit)
65+
66+
def next(newPageSize: Int): F[RequestResult[GroupedPage[K, V, F]]] = {
67+
if (hasNext) {
68+
pageRoller(newPageSize, pageStart + limit)
69+
} else {
70+
MonadError[F].unit(RequestFail(NoDataException("No next page")))
71+
}
72+
}
73+
74+
def next: F[RequestResult[GroupedPage[K, V, F]]] = next(limit)
75+
76+
def +(other: GroupedPage[K, V, F]): GroupedPage[K, V, F] = {
77+
val newItems = other.items.foldLeft(items) { case (acc, (k, v)) =>
78+
if (acc.contains(k)) {
79+
acc.updated(k, acc(k) ++ v)
80+
} else {
81+
acc + (k -> v)
82+
}
83+
}
84+
val newHasNext = hasNext && other.hasNext
85+
val newPageStart = pageStart min other.pageStart
86+
val newPageEnd = pageEnd max other.pageEnd
87+
this.copy(items = newItems, hasNext = newHasNext, pageStart = newPageStart, pageEnd = newPageEnd)
88+
}
89+
}
90+
91+
object GroupedPage {
92+
type GroupPageRoller[K, V, F[_]] = (Int, Long) => F[RequestResult[GroupedPage[K, V, F]]]
93+
}
Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Copyright 2021 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package za.co.absa.atum.reader.result
18+
19+
import sttp.monad.MonadError
20+
import sttp.monad.syntax._
21+
import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestPageResultOps, RequestResult}
22+
import za.co.absa.atum.reader.exceptions.RequestException.NoDataException
23+
import za.co.absa.atum.reader.result.GroupedPage.GroupPageRoller
24+
import za.co.absa.atum.reader.result.Page.PageRoller
25+
26+
import scala.collection.immutable.ListMap
27+
28+
case class Page[T, F[_]: MonadError](
29+
items: Vector[T],
30+
hasNext: Boolean,
31+
limit: Int,
32+
pageStart: Long,
33+
pageEnd: Long,
34+
private[reader] val pageRoller: PageRoller[T, F]
35+
) extends AbstractPage[Vector[T], F] {
36+
37+
def apply(index: Int): T = items(index)
38+
39+
def map[B](f: T => B): Page[B, F] = {
40+
val newItems = items.map(f)
41+
val newPageRoller: PageRoller[B, F] = (limit, offset) => pageRoller(limit, offset).map(_.pageMap(f))
42+
this.copy(items = newItems, pageRoller = newPageRoller)
43+
}
44+
45+
def prior(newPageSize: Int): F[RequestResult[Page[T, F]]] = {
46+
if (hasPrior) {
47+
val newOffset = (pageStart - newPageSize).max(0)
48+
pageRoller(newPageSize, newOffset)
49+
} else {
50+
MonadError[F].unit(RequestFail(NoDataException("No prior page")))
51+
}
52+
}
53+
54+
def prior(): F[RequestResult[Page[T, F]]] = prior(limit)
55+
56+
def next(newPageSize: Int): F[RequestResult[Page[T, F]]] = {
57+
if (hasNext) {
58+
pageRoller(newPageSize, pageStart + pageSize)
59+
} else {
60+
MonadError[F].unit(RequestFail(NoDataException("No next page")))
61+
}
62+
}
63+
64+
def next: F[RequestResult[Page[T, F]]] = next(limit)
65+
66+
def +(other: Page[T, F]): Page[T, F] = {
67+
val newItems = items ++ other.items
68+
val newPageStart = pageStart min other.pageStart
69+
val newPageEnd = pageEnd max other.pageEnd
70+
val newHasNext = hasNext && other.hasNext
71+
this.copy(items = newItems, hasNext = newHasNext, pageStart = newPageStart, pageEnd = newPageEnd)
72+
}
73+
74+
def groupBy[K](f: T => K): GroupedPage[K, T, F] = {
75+
val (newItems, itemsCounts) = items.foldLeft(ListMap.empty[K, Vector[T]], 0) { case ((groupsAcc, count), item) =>
76+
val k = f(item)
77+
(groupsAcc.updated(k, groupsAcc.getOrElse(k, Vector.empty) :+ item), count + 1)
78+
}
79+
val newPageRoller: GroupPageRoller[K, T, F] = (limit, offset) =>
80+
pageRoller(limit, offset)
81+
.map(_.map(_.groupBy(f)))
82+
83+
GroupedPage(
84+
newItems,
85+
hasNext,
86+
limit,
87+
pageStart,
88+
pageEnd,
89+
newPageRoller
90+
)
91+
}
92+
}
93+
94+
object Page {
95+
type PageRoller[T, F[_]] = (Int, Long) => F[RequestResult[Page[T, F]]]
96+
}
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Copyright 2021 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package za.co.absa.atum.reader.result
18+
19+
import org.scalatest.funsuite.AnyFunSuiteLike
20+
import sttp.client3.Identity
21+
import sttp.client3.monad.IdMonad
22+
import sttp.monad.MonadError
23+
24+
25+
class AbstractPageUnitTests extends AnyFunSuiteLike {
26+
private implicit val monad: MonadError[Identity] = IdMonad
27+
28+
test("Basic test") {
29+
val page = new AbstractPage[Iterable[Int], Identity] {
30+
override def items: Iterable[Int] = Seq(1, 2, 3)
31+
override def hasNext: Boolean = true
32+
override def limit: Int = 3
33+
override def pageStart: Long = 0
34+
override def pageEnd: Long = 2
35+
}
36+
37+
assert(page.items.size == 3)
38+
assert(page.hasNext)
39+
assert(page.limit == 3)
40+
assert(page.pageStart == 0)
41+
assert(page.pageSize == 3)
42+
assert(!page.hasPrior)
43+
44+
val anotherPage = new AbstractPage[Iterable[Int], Identity] {
45+
override def items: Iterable[Int] = Seq(1, 2, 3)
46+
override def hasNext: Boolean = true
47+
override def limit: Int = 3
48+
override def pageStart: Long = 1
49+
override def pageEnd: Long = 2
50+
}
51+
assert(anotherPage.hasPrior)
52+
}
53+
}

0 commit comments

Comments
 (0)