Feauture/#40 info fields (#87)

close #40

Author: angelcervera

.gitignore

@@ -2,3 +2,4 @@
 target
 core/src/pbf_generated/
 project/project/
+tmp

README.md

@@ -57,7 +57,3 @@ export GIT_USER=<username>; export USE_SSH=true; npm run deploy
 
 ### third party OSS libraries:
   - ScalaPB: https://scalapb.github.io/ and https://github.com/thesamet/sbt-protoc
-
-
-```
-

build.sbt

@@ -11,21 +11,21 @@ lazy val commonIOVersion = "2.5"
 lazy val logbackVersion = "1.1.7"
 lazy val scoptVersion = "3.7.1"
 lazy val akkaVersion = "2.5.31"
-lazy val spark3Version = "3.0.1"
+lazy val spark3Version = "3.1.1"
 lazy val spark2Version = "2.4.7"
 lazy val sparkDefaultVersion = spark3Version
 
-lazy val scala213 = "2.13.3"
-lazy val scala212 = "2.12.12"
+lazy val scala213 = "2.13.5"
+lazy val scala212 = "2.12.13"
 lazy val scala211 = "2.11.12"
 lazy val scalaVersions = if (isPatch211Enable()) Seq(scala211) else Seq(scala213, scala212)
 lazy val sparkScalaVersions = if (isPatch211Enable()) Seq(scala211) else Seq(scala212)
 
 lazy val commonSettings = Seq(
   crossScalaVersions := scalaVersions,
   organization := "com.acervera.osm4scala",
-  organizationHomepage := Some(url("http://www.acervera.com")),
-  licenses += ("MIT", url("http://opensource.org/licenses/MIT")),
+  organizationHomepage := Some(url("https://www.acervera.com")),
+  licenses += ("MIT", url("https://opensource.org/licenses/MIT")),
   homepage in ThisBuild := Some(
     url(s"https://simplexspatial.github.io/osm4scala/")
   ),
@@ -116,7 +116,7 @@ def generateSparkModule(sparkVersion: String): Project = {
   def pathFromModule(relativePath: String): String = if (sparkDefaultVersion == sparkVersion) {
     relativePath
   } else {
-    s"../../spark/${relativePath}"
+    s"../../spark/$relativePath"
   }
 
   Project(id = s"spark${sparkVersion.head}", base = file(baseFolder))
@@ -169,6 +169,8 @@ def listOfProjects(): Seq[ProjectReference] = {
     examplesCounterAkka,
     examplesTagsExtraction,
     examplesPrimitivesExtraction,
+    examplesBlocksExtraction,
+    examplesTakeN
   )
 
   val spark3Projects: Seq[ProjectReference] = Seq(
@@ -180,7 +182,7 @@ def listOfProjects(): Seq[ProjectReference] = {
 
   val projects = modules ++ (if(isPatch211Enable()) Seq.empty else spark3Projects)
 
-  print(s"PATCH_211 is ${isPatch211Enable()} so we are going to work with this list of projects: \n${projects.mkString("\t- ", "\n\t- ", "")}")
+  println(s"PATCH_211 is ${isPatch211Enable()} so we are going to work with this list of projects: \n${projects.mkString("\t- ", "\n\t- ", "")}")
 
   projects
 }
@@ -293,6 +295,16 @@ lazy val examplesBlocksExtraction = Project(id = "examples-blocks-extraction", b
   )
   .dependsOn(core, commonUtilities)
 
+lazy val examplesTakeN = Project(id = "examples-takeN", base = file("examples/takeN"))
+  .disablePlugins(AssemblyPlugin)
+  .settings(
+    commonSettings,
+    exampleSettings,
+    name := "osm4scala-examples-takeN",
+    description := "Generate a pbf file by taking the first N blocks."
+  )
+  .dependsOn(core, commonUtilities)
+
 lazy val examplesPrimitivesExtraction =
   Project(id = "examples-primitives-extraction", base = file("examples/primitivesextraction"))
     .disablePlugins(AssemblyPlugin)

core/src/main/scala/com/acervera/osm4scala/DenseNodesIterator.scala

@@ -25,9 +25,11 @@
 
 package com.acervera.osm4scala
 
-import com.acervera.osm4scala.model.NodeEntity
-import com.acervera.osm4scala.utilities.StringTableUtils
-import org.openstreetmap.osmosis.osmbinary.osmformat.{DenseNodes, StringTable}
+import com.acervera.osm4scala.model.{Info, NodeEntity}
+import com.acervera.osm4scala.utilities.StringTableUtils._
+import org.openstreetmap.osmosis.osmbinary.osmformat.{DenseInfo, DenseNodes, StringTable}
+
+import java.time.Instant
 
 object DenseNodesIterator {
 
@@ -51,7 +53,7 @@ class DenseNodesIterator(osmosisStringTable: StringTable,
                          latOffset: Long = 0,
                          lonOffset: Long = 0,
                          granularity: Int = 100)
-    extends Iterator[NodeEntity] with StringTableUtils {
+    extends Iterator[NodeEntity] {
 
   if (osmosisDenseNode.denseinfo.isDefined && osmosisDenseNode.denseinfo.get.visible.nonEmpty) {
     throw new Exception("Only visible nodes are implemented.")
@@ -61,28 +63,31 @@ class DenseNodesIterator(osmosisStringTable: StringTable,
   private val lonIterator = osmosisDenseNode.lon.iterator
   private val latIterator = osmosisDenseNode.lat.iterator
   private val tagsIterator = osmosisDenseNode.keysVals.iterator
+  private val infoIterator = InfoIterator(osmosisDenseNode.denseinfo)
 
-  private var lastNode: NodeEntity = NodeEntity(0, 0, 0, Map())
+  // Delta references
+  var lastId = 0L
+  var lastLatitude = 0.0
+  var lastLongitude = 0.0
 
   override def hasNext: Boolean = idIterator.hasNext
 
   override def next(): NodeEntity = {
 
-    val id = idIterator.next() + lastNode.id
-    val latitude =
-      decompressCoord(latOffset, latIterator.next(), lastNode.latitude)
-    val longitude =
-      decompressCoord(lonOffset, lonIterator.next(), lastNode.longitude)
+    // Calculate new values base in deltas and update deltas
+    lastId = idIterator.next() + lastId
+    lastLatitude = decompressCoord(latOffset, latIterator.next(), lastLatitude)
+    lastLongitude = decompressCoord(lonOffset, lonIterator.next(), lastLongitude)
 
     // Create node
-    lastNode = NodeEntity(
-      id,
-      latitude,
-      longitude,
-      osmosisStringTable.extractTags(tagsIterator.takeWhile(_ != 0L))
+    NodeEntity(
+      lastId,
+      lastLatitude,
+      lastLongitude,
+      osmosisStringTable.extractTags(tagsIterator.takeWhile(_ != 0L)),
+      infoIterator.next()
     )
 
-    lastNode
   }
 
   /**
@@ -97,4 +102,84 @@ class DenseNodesIterator(osmosisStringTable: StringTable,
     (.000000001 * (offSet + (granularity * delta))) + currentValue
   }
 
+  // Decode DenseInfo
+
+  trait InfoIterator extends Iterator[Option[Info]]
+
+  class InfoIteratorDeltas(denseInfo: DenseInfo) extends InfoIterator {
+    val versionIterator = denseInfo.version.iterator
+    val timestampIterator = denseInfo.timestamp.iterator
+    val changesetIterator = denseInfo.changeset.iterator
+    val uidIterator = denseInfo.uid.iterator
+    val userNameIterator = denseInfo.userSid.iterator
+    val visibleIterator = denseInfo.visible.iterator
+
+    var lastTimestamp = 0L
+    var lastChangeset = 0L
+    var lastUserId = 0
+    var lastUserName = 0
+
+    override def hasNext: Boolean =
+      versionIterator.hasNext ||
+        timestampIterator.hasNext ||
+        changesetIterator.hasNext ||
+        uidIterator.hasNext ||
+        userNameIterator.hasNext ||
+        visibleIterator.hasNext
+
+    override def next(): Option[Info] = {
+
+      val newTimestamp = if (timestampIterator.hasNext) {
+        lastTimestamp = lastTimestamp + timestampIterator.next()
+        Some(Instant.ofEpochSecond(lastTimestamp))
+      } else {
+        None
+      }
+
+      val newChangeset = if (changesetIterator.hasNext) {
+        lastChangeset = lastChangeset + changesetIterator.next()
+        Some(lastChangeset)
+      } else {
+        None
+      }
+
+      val newUserId = if (uidIterator.hasNext) {
+        lastUserId = lastUserId + uidIterator.next()
+        Some(lastUserId)
+      } else {
+        None
+      }
+
+      val newUserName = if (userNameIterator.hasNext) {
+        lastUserName = lastUserName + userNameIterator.next()
+        Some(osmosisStringTable.getString(lastUserName))
+      } else {
+        None
+      }
+
+      val info = Info(
+        version = if (versionIterator.hasNext) Some(versionIterator.next()) else None,
+        timestamp = newTimestamp,
+        changeset = newChangeset,
+        userId = newUserId,
+        userName = newUserName,
+        visible = if (visibleIterator.hasNext) Some(visibleIterator.next()) else None
+      )
+
+      Some(info)
+    }
+  }
+
+  object InfoIterator {
+    def apply(denseInfo: Option[DenseInfo]): InfoIterator = denseInfo match {
+      case None       => InfoIteratorNone
+      case Some(info) => new InfoIteratorDeltas(info)
+    }
+  }
+
+  object InfoIteratorNone extends InfoIterator {
+    override def next(): Option[Info] = None
+    override def hasNext: Boolean = true
+  }
+
 }