Skip to content

Commit ca535fc

Browse files
committed
[wip] Refactor ParquetType to pass Configuration at instantiation time
1 parent 56ff232 commit ca535fc

File tree

5 files changed

+170
-124
lines changed

5 files changed

+170
-124
lines changed

Diff for: jmh/src/test/scala/magnolify/jmh/MagnolifyBench.scala

+23-15
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,14 @@ import magnolify.jmh.MagnolifyBench.nested
2020
import magnolify.parquet.{MagnolifyParquetProperties, ParquetType, TestInputFile, TestOutputFile}
2121

2222
import java.util.concurrent.TimeUnit
23-
import magnolify.scalacheck.auto.*
24-
import magnolify.test.Simple.*
23+
import magnolify.scalacheck.auto._
24+
import magnolify.test.Simple._
2525
import org.apache.hadoop.conf.Configuration
2626
import org.apache.parquet.hadoop.{ParquetReader, ParquetWriter}
27-
import org.scalacheck.*
28-
import org.openjdk.jmh.annotations.*
27+
import org.scalacheck._
28+
import org.openjdk.jmh.annotations._
2929

30-
import scala.jdk.CollectionConverters.*
30+
import scala.jdk.CollectionConverters._
3131

3232
object MagnolifyBench {
3333
val seed: rng.Seed = rng.Seed(0)
@@ -94,7 +94,7 @@ class AvroBench {
9494
}
9595

9696
@State(Scope.Benchmark)
97-
class ParquetReadState()(implicit pt: ParquetType[Nested]) {
97+
class ParquetReadState(pt: ParquetType[Nested]) {
9898
var out: TestOutputFile = null
9999
var reader: ParquetReader[Nested] = null
100100

@@ -116,7 +116,7 @@ class ParquetReadState()(implicit pt: ParquetType[Nested]) {
116116
}
117117

118118
@State(Scope.Benchmark)
119-
class ParquetWriteState()(implicit pt: ParquetType[Nested]) {
119+
class ParquetWriteState(pt: ParquetType[Nested]) {
120120
var writer: ParquetWriter[Nested] = null
121121

122122
@Setup(Level.Invocation)
@@ -131,29 +131,37 @@ class ParquetWriteState()(implicit pt: ParquetType[Nested]) {
131131
}
132132
}
133133

134+
object ParquetStates {
135+
def confWithGroupedArraysProp(propValue: Boolean): Configuration = {
136+
val conf = new Configuration()
137+
conf.setBoolean(MagnolifyParquetProperties.WriteGroupedArrays, propValue)
138+
conf
139+
}
140+
class DefaultParquetReadState extends ParquetReadState(ParquetType[Nested](confWithGroupedArraysProp(false)))
141+
class DefaultParquetWriteState extends ParquetWriteState(ParquetType[Nested](confWithGroupedArraysProp(false)))
142+
143+
class ParquetAvroCompatReadState extends ParquetReadState(ParquetType[Nested](confWithGroupedArraysProp(true)))
144+
class ParquetAvroCompatWriteState extends ParquetWriteState(ParquetType[Nested](confWithGroupedArraysProp(true)))
145+
}
134146

135147
@BenchmarkMode(Array(Mode.AverageTime))
136148
@OutputTimeUnit(TimeUnit.NANOSECONDS)
137149
@State(Scope.Thread)
138150
class ParquetBench {
139151
import MagnolifyBench._
140-
implicit val pt: ParquetType[Nested] = ParquetType[Nested]
141152

142-
@Benchmark def parquetWrite(state: ParquetWriteState): Unit = state.writer.write(nested)
143-
@Benchmark def parquetRead(state: ParquetReadState): Nested = state.reader.read()
153+
@Benchmark def parquetWrite(state: ParquetStates.DefaultParquetWriteState): Unit = state.writer.write(nested)
154+
@Benchmark def parquetRead(state: ParquetStates.DefaultParquetReadState): Nested = state.reader.read()
144155
}
145156

146157
@BenchmarkMode(Array(Mode.AverageTime))
147158
@OutputTimeUnit(TimeUnit.NANOSECONDS)
148159
@State(Scope.Thread)
149160
class ParquetAvroCompatBench {
150161
import MagnolifyBench._
151-
val conf = new Configuration()
152-
conf.setBoolean(MagnolifyParquetProperties.WriteGroupedArrays, MagnolifyParquetProperties.WriteGroupedArraysDefault)
153-
implicit val pt: ParquetType[Nested] = ParquetType[Nested](conf)
154162

155-
@Benchmark def parquetWrite(state: ParquetWriteState): Unit = state.writer.write(nested)
156-
@Benchmark def parquetRead(state: ParquetReadState): Nested = state.reader.read()
163+
@Benchmark def parquetWrite(state: ParquetStates.ParquetAvroCompatWriteState): Unit = state.writer.write(nested)
164+
@Benchmark def parquetRead(state: ParquetStates.ParquetAvroCompatReadState): Nested = state.reader.read()
157165
}
158166

159167
@BenchmarkMode(Array(Mode.AverageTime))

0 commit comments

Comments
 (0)