Skip to content

Commit b9188a0

Browse files
authored
Fix timestamp spark calculation (#101)
* Fix timestamp spark calculation
1 parent 545d924 commit b9188a0

File tree

2 files changed

+17
-1
lines changed

2 files changed

+17
-1
lines changed

spark/src/main/scala/com/acervera/osm4scala/spark/OsmPbfRowIterator.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ object OsmPbfRowIterator {
8080

8181
private def populateInfo(info: Info): InternalRow = InternalRow.fromSeq(infoSchema.fieldNames.map{
8282
case FIELD_INFO_VERSION => info.version.getOrElse(null)
83-
case FIELD_INFO_TIMESTAMP => info.timestamp.map(inst => inst.toEpochMilli).getOrElse(null)
83+
case FIELD_INFO_TIMESTAMP => info.timestamp.map(inst => inst.toEpochMilli * 1000).getOrElse(null)
8484
case FIELD_INFO_CHANGESET => info.changeset.getOrElse(null)
8585
case FIELD_INFO_USER_ID => info.userId.getOrElse(null)
8686
case FIELD_INFO_USER_NAME => info.userName.map(UTF8String.fromString).orNull

spark/src/test/scala/com/acervera/osm4scala/spark/OsmPbfFormatSpec.scala

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import org.scalatest.prop.TableDrivenPropertyChecks
3636
import org.scalatest.wordspec.AnyWordSpec
3737

3838
import java.io.File
39+
import java.sql.Timestamp
3940
import scala.util.Random
4041

4142
object SourcesForTesting {
@@ -145,6 +146,21 @@ class OsmPbfFormatSpec extends AnyWordSpec with Matchers with SparkSessionBefore
145146
node171946.getAs[Long]("id") shouldBe 171946L
146147
}
147148

149+
"read info" in {
150+
// <node id="1699777711" version="2" timestamp="2018-03-26T07:24:26Z" lat="43.7402163" lon="7.4281505"/>
151+
// Epoch 1522049066000 = Monday, March 26, 2018 7:24:26 AM
152+
val node1699777711 = loadOsmPbf(spark, monacoPath)
153+
.select(
154+
col("id"),
155+
col("info.version") as "version",
156+
col("info.timestamp") as "timestamp"
157+
).filter("id == 1699777711").collect()(0)
158+
159+
node1699777711.getAs[Long]("id") shouldBe 1699777711L
160+
node1699777711.getAs[Integer]("version") should be(2)
161+
node1699777711.getAs[Timestamp]("timestamp").toInstant.toEpochMilli shouldBe 1522049066000L
162+
}
163+
148164
"read null info" in {
149165
val node171946 = loadOsmPbf(spark, madridPath).select("id", "info").filter("id == 171946").collect()(0)
150166
node171946.getAs[Long]("id") shouldBe 171946L

0 commit comments

Comments
 (0)