Skip to content

Commit 82ba554

Browse files
caiiiycukcaiiiycuk
committed
v1.1.0: date_class support
1 parent 6e8b500 commit 82ba554

File tree

6 files changed

+117
-22
lines changed

6 files changed

+117
-22
lines changed

‎project/Build.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ object Build extends Build {
77
lazy val project = Project("root", file("."), settings = Seq(
88
name := "postgresql-to-sqlite",
99
organization := "com.github.caiiiycuk",
10-
version := "1.0.3",
10+
version := "1.1.0",
1111
scalaVersion := "2.11.12",
1212

1313
libraryDependencies ++= Seq(

‎src/main/scala/com/github/caiiiycuk/pg2sqlite/Boot.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ object Boot extends App with Log {
1515
import config._
1616

1717
val size = pgdump.length()
18-
val connection = Connection.sqlite(sqlite)
18+
val connection = Connection.sqlite(sqlite, config.dateClass)
1919
val iterator = LineIterator(pgdump)
2020
val loggedIterator = LoggedIterator(iterator, () => 100.0 * iterator.readed / size)
2121
val dumpInserter = new DumpInserter(connection)

‎src/main/scala/com/github/caiiiycuk/pg2sqlite/Config.scala

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,34 @@ package com.github.caiiiycuk.pg2sqlite
22

33
import java.io.File
44

5-
case class Config(pgdump: File = new File("dump"), sqlite: File = new File("db"), force: Boolean = false)
5+
case class Config(pgdump: File = new File("dump"), sqlite: File = new File("db"),
6+
force: Boolean = false, dateClass: String = Connection.DEFAULT_DATE_CLASS)
67

78
object Config extends Log {
89
private val parser = new scopt.OptionParser[Config]("postgresql-to-sqlite") {
910
head("postgresql-to-sqlite")
1011

11-
opt[File]('d', "dump") required () valueName ("<dump file>") action { (v, c) =>
12+
opt[File]('d', "dump") required() valueName ("<dump file>") action { (v, c) =>
1213
c.copy(pgdump = v)
1314
} text ("postgresql dump generated by pg_dump")
1415

15-
opt[File]('o', "out") required () valueName ("<sqlite3 database>") action { (v, c) =>
16+
opt[File]('o', "out") required() valueName ("<sqlite3 database>") action { (v, c) =>
1617
c.copy(sqlite = v)
1718
} text ("sqlite3 database to create")
1819

19-
opt[Boolean]('f', "force") optional () valueName ("<true|false>") action { (v, c) =>
20+
opt[Boolean]('f', "force") optional() valueName ("<true|false>") action { (v, c) =>
2021
c.copy(force = v)
2122
} text ("recreate database if exists")
2223

24+
opt[String]('t', "timestamps") optional() valueName ("<integer|text|real>") action { (v, c) =>
25+
val dc = v.toUpperCase()
26+
if (dc.equals(Connection.TEXT_DATE_CLASS) || dc.equals(Connection.REAL_DATE_CLASS)) {
27+
c.copy(dateClass = dc)
28+
} else {
29+
c
30+
}
31+
} text ("Change sqlite3 date class (default: INTEGER)")
32+
2333
checkConfig { c =>
2434
import c._
2535

‎src/main/scala/com/github/caiiiycuk/pg2sqlite/Connection.scala

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,36 @@
11
package com.github.caiiiycuk.pg2sqlite
22

3+
import org.sqlite.SQLiteConfig
4+
35
import java.sql.DriverManager
46
import java.sql.Statement
57
import java.sql.PreparedStatement
68
import scala.collection.mutable.ListBuffer
79
import java.sql.ResultSet
810
import scala.annotation.tailrec
911
import java.io.File
12+
import java.util.Properties
1013

1114
trait ConnectionHolder {
1215
def makeConnection: java.sql.Connection
16+
1317
def db: String
1418
}
1519

1620
object Connection {
17-
final val FETCH_SIZE = 8192
18-
final val MAX_VARIABLE_NUMBER = 999
19-
20-
def sqlite(dbFile: File) = {
21+
final val DEFAULT_DATE_CLASS = "INTEGER"
22+
final val TEXT_DATE_CLASS = "TEXT"
23+
final val REAL_DATE_CLASS = "REAL"
24+
private final val DATE_CLASS_PRAGMA = "date_class"
25+
private final val FETCH_SIZE = 8192
26+
private final val MAX_VARIABLE_NUMBER = 999
27+
28+
def sqlite(dbFile: File, dateClass: String = DEFAULT_DATE_CLASS): Connection = {
2129
val connectionHolder = new ConnectionHolder {
2230
override def makeConnection: java.sql.Connection = {
23-
implicit val connection = DriverManager.getConnection(s"jdbc:sqlite:$dbFile")
31+
val properties = new Properties()
32+
properties.setProperty(DATE_CLASS_PRAGMA, dateClass)
33+
implicit val connection = DriverManager.getConnection(s"jdbc:sqlite:$dbFile", properties)
2434

2535
connection.setAutoCommit(true)
2636
sqlitePragmas()
@@ -36,14 +46,15 @@ object Connection {
3646
}
3747

3848
private def sqlitePragmas()(implicit connection: java.sql.Connection) = {
39-
val statment = connection.createStatement()
40-
statment.executeUpdate("PRAGMA synchronous = OFF")
41-
statment.executeUpdate("PRAGMA journal_mode = OFF")
42-
statment.executeUpdate("PRAGMA threads = 64")
43-
statment.executeUpdate("PRAGMA max_page_count = 2147483646")
44-
statment.executeUpdate("PRAGMA cache_size = 65536")
45-
statment.executeUpdate("PRAGMA cache_spill = true")
46-
statment.close
49+
assert(SQLiteConfig.Pragma.DATE_CLASS.pragmaName.equals(DATE_CLASS_PRAGMA));
50+
val statement = connection.createStatement()
51+
statement.executeUpdate(s"PRAGMA ${SQLiteConfig.Pragma.SYNCHRONOUS.pragmaName} = OFF")
52+
statement.executeUpdate(s"PRAGMA ${SQLiteConfig.Pragma.JOURNAL_MODE.pragmaName} = OFF")
53+
statement.executeUpdate(s"PRAGMA ${SQLiteConfig.Pragma.LIMIT_WORKER_THREADS.pragmaName} = 64")
54+
statement.executeUpdate(s"PRAGMA ${SQLiteConfig.Pragma.MAX_PAGE_COUNT.pragmaName} = 2147483646")
55+
statement.executeUpdate(s"PRAGMA ${SQLiteConfig.Pragma.CACHE_SIZE.pragmaName} = 65536")
56+
statement.executeUpdate("PRAGMA cache_spill = true")
57+
statement.close
4758
}
4859
}
4960

‎src/main/scala/com/github/caiiiycuk/pg2sqlite/DumpInserter.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ class DumpInserter(connection: Connection) {
2121
val head = iterator.next()
2222
val fullIterator = Iterator(head) ++ iterator
2323

24-
COMMANDS.find(_.matchHead(head)).map { command =>
24+
COMMANDS.find(_.matchHead(head)).foreach { command =>
2525
command.apply(connection, fullIterator)
2626
}
2727

‎src/test/scala/com/github/caiiiycuk/pg2sqlite/dsl/DumperTest.scala

Lines changed: 76 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.github.caiiiycuk.pg2sqlite.dsl
22

3+
import org.scalatest.FlatSpec
4+
import org.scalatest.Matchers
35
import com.github.caiiiycuk.pg2sqlite.iterator.Line
46
import com.github.caiiiycuk.pg2sqlite.{Connection, DumpInserter}
57
import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers}
@@ -10,12 +12,23 @@ class DumperTest extends FlatSpec with Matchers with BeforeAndAfter {
1012

1113
val dbFile = new File("test.db")
1214

13-
private def makeConnection() = {
15+
private final val DATE_DUMP =
16+
"""
17+
|CREATE TABLE test (
18+
| current timestamp without time zone NOT NULL
19+
|);
20+
|
21+
|COPY test (current) FROM stdin;
22+
|2024-05-06 15:14:12
23+
|\.
24+
|""".stripMargin
25+
26+
private def makeConnection(dateClass: String = Connection.DEFAULT_DATE_CLASS) = {
1427
if (dbFile.exists()) {
1528
dbFile.delete()
1629
}
1730

18-
Connection.sqlite(dbFile)
31+
Connection.sqlite(dbFile, dateClass)
1932
}
2033

2134
after {
@@ -54,4 +67,65 @@ class DumperTest extends FlatSpec with Matchers with BeforeAndAfter {
5467
inserter.insert(dump.iterator)
5568
connection.close
5669
}
70+
71+
"dumper" should "should respect date class (Default)" in {
72+
val connection = makeConnection()
73+
val inserter = new DumpInserter(connection)
74+
val dump = DATE_DUMP.split("\n")
75+
.zipWithIndex
76+
.map {
77+
case (text, num) =>
78+
Line(num, text)
79+
}
80+
81+
inserter.insert(dump.iterator)
82+
connection.withStatement { statment =>
83+
val rs = statment.executeQuery("SELECT * FROM test")
84+
rs.next() should equal(true)
85+
rs.getString(1) should equal("1714983252000")
86+
rs.close()
87+
}
88+
connection.close
89+
}
90+
91+
"dumper" should "should respect date class (text)" in {
92+
val connection = makeConnection(Connection.TEXT_DATE_CLASS)
93+
val inserter = new DumpInserter(connection)
94+
val dump = DATE_DUMP.split("\n")
95+
.zipWithIndex
96+
.map {
97+
case (text, num) =>
98+
Line(num, text)
99+
}
100+
101+
inserter.insert(dump.iterator)
102+
connection.withStatement { statment =>
103+
val rs = statment.executeQuery("SELECT * FROM test")
104+
rs.next() should equal(true)
105+
rs.getString(1) should equal("2024-05-06 15:14:12.000")
106+
rs.close()
107+
}
108+
connection.close
109+
}
110+
111+
"dumper" should "should respect date class (real)" in {
112+
val connection = makeConnection(Connection.REAL_DATE_CLASS)
113+
val inserter = new DumpInserter(connection)
114+
val dump = DATE_DUMP.split("\n")
115+
.zipWithIndex
116+
.map {
117+
case (text, num) =>
118+
Line(num, text)
119+
}
120+
121+
inserter.insert(dump.iterator)
122+
connection.withStatement { statment =>
123+
val rs = statment.executeQuery("SELECT * FROM test")
124+
rs.next() should equal(true)
125+
rs.getString(1) should equal("2460436.84319444")
126+
rs.close()
127+
}
128+
connection.close
129+
}
130+
57131
}

0 commit comments

Comments
 (0)