Skip to content

Commit a287fb6

Browse files
caiiiycukcaiiiycuk
committed
v0.1-SNAPSHOT
1 parent 05079c8 commit a287fb6

26 files changed

+1184
-1
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,4 @@ project/plugins/project/
1515
# Scala-IDE specific
1616
.scala_dependencies
1717
.worksheet
18+
/bin/

‎.travis.yml

+5-1
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,8 @@ jdk:
88
- oraclejdk7
99
- oraclejdk8
1010
- openjdk7
11-
- openjdk8
11+
12+
script:
13+
- sbt ++$TRAVIS_SCALA_VERSION clean
14+
- sbt ++$TRAVIS_SCALA_VERSION test
15+
- sbt ++$TRAVIS_SCALA_VERSION one-jar

‎project/Build.scala

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import com.github.retronym.SbtOneJar
2+
import sbt._
3+
import Keys._
4+
5+
object Build extends Build {
6+
7+
lazy val project = Project("root", file("."), settings = Seq(
8+
name := "postgresql-to-sqlite",
9+
organization := "com.github.caiiiycuk",
10+
version := "0.0.1-SNAPSHOT",
11+
scalaVersion := "2.11.7",
12+
13+
libraryDependencies ++= Seq(
14+
"com.github.scopt" %% "scopt" % "3.3.0",
15+
"ch.qos.logback" % "logback-classic" % "1.1.2",
16+
"org.xerial" % "sqlite-jdbc" % "3.8.10.2",
17+
"org.scalatest" %% "scalatest" % "2.2.4" % "test"
18+
)
19+
) ++ SbtOneJar.oneJarSettings)
20+
21+
}

‎project/build.properties

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
sbt.version=0.13.5

‎project/plugins.sbt

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "4.0.0")
2+
3+
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.7.0")
4+
5+
addSbtPlugin("org.scala-sbt.plugins" % "sbt-onejar" % "0.8")

‎scalastyle-config.xml

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
<scalastyle>
2+
<name>Scalastyle standard configuration</name>
3+
<check level="error" class="org.scalastyle.file.FileTabChecker" enabled="true"></check>
4+
<check level="error" class="org.scalastyle.file.FileLengthChecker" enabled="true">
5+
<parameters>
6+
<parameter name="maxFileLength"><![CDATA[800]]></parameter>
7+
</parameters>
8+
</check>
9+
<check level="error" class="org.scalastyle.scalariform.SpacesAfterPlusChecker" enabled="true"></check>
10+
<check level="error" class="org.scalastyle.file.WhitespaceEndOfLineChecker" enabled="true"></check>
11+
<check level="error" class="org.scalastyle.scalariform.SpacesBeforePlusChecker" enabled="true"></check>
12+
<check level="error" class="org.scalastyle.file.FileLineLengthChecker" enabled="true">
13+
<parameters>
14+
<parameter name="maxLineLength"><![CDATA[160]]></parameter>
15+
<parameter name="tabSize"><![CDATA[4]]></parameter>
16+
</parameters>
17+
</check>
18+
<check level="error" class="org.scalastyle.scalariform.ClassNamesChecker" enabled="true">
19+
<parameters>
20+
<parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter>
21+
</parameters>
22+
</check>
23+
<check level="error" class="org.scalastyle.scalariform.ObjectNamesChecker" enabled="true">
24+
<parameters>
25+
<parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter>
26+
</parameters>
27+
</check>
28+
<check level="error" class="org.scalastyle.scalariform.PackageObjectNamesChecker" enabled="true">
29+
<parameters>
30+
<parameter name="regex"><![CDATA[^[a-z][A-Za-z]*$]]></parameter>
31+
</parameters>
32+
</check>
33+
<check level="error" class="org.scalastyle.scalariform.EqualsHashCodeChecker" enabled="true"></check>
34+
<check level="error" class="org.scalastyle.scalariform.IllegalImportsChecker" enabled="true">
35+
<parameters>
36+
<parameter name="illegalImports"><![CDATA[sun._,java.awt._]]></parameter>
37+
</parameters>
38+
</check>
39+
<check level="error" class="org.scalastyle.scalariform.ParameterNumberChecker" enabled="true">
40+
<parameters>
41+
<parameter name="maxParameters"><![CDATA[8]]></parameter>
42+
</parameters>
43+
</check>
44+
<check level="error" class="org.scalastyle.scalariform.MagicNumberChecker" enabled="true">
45+
<parameters>
46+
<parameter name="ignore"><![CDATA[-1,0,1,2,3]]></parameter>
47+
</parameters>
48+
</check>
49+
<check level="error" class="org.scalastyle.scalariform.NoWhitespaceBeforeLeftBracketChecker" enabled="true"></check>
50+
<check level="error" class="org.scalastyle.scalariform.NoWhitespaceAfterLeftBracketChecker" enabled="true"></check>
51+
<check level="error" class="org.scalastyle.scalariform.ReturnChecker" enabled="true"></check>
52+
<check level="error" class="org.scalastyle.scalariform.NullChecker" enabled="true"></check>
53+
<check level="error" class="org.scalastyle.scalariform.NoCloneChecker" enabled="true"></check>
54+
<check level="error" class="org.scalastyle.scalariform.NoFinalizeChecker" enabled="true"></check>
55+
<check level="error" class="org.scalastyle.scalariform.CovariantEqualsChecker" enabled="true"></check>
56+
<check level="error" class="org.scalastyle.scalariform.StructuralTypeChecker" enabled="true"></check>
57+
<check level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
58+
<parameters>
59+
<parameter name="regex"><![CDATA[println]]></parameter>
60+
</parameters>
61+
</check>
62+
<check level="error" class="org.scalastyle.scalariform.NumberOfTypesChecker" enabled="true">
63+
<parameters>
64+
<parameter name="maxTypes"><![CDATA[30]]></parameter>
65+
</parameters>
66+
</check>
67+
<check level="error" class="org.scalastyle.scalariform.CyclomaticComplexityChecker" enabled="false">
68+
<parameters>
69+
<parameter name="maximum"><![CDATA[10]]></parameter>
70+
</parameters>
71+
</check>
72+
<check level="error" class="org.scalastyle.scalariform.UppercaseLChecker" enabled="true"></check>
73+
<check level="error" class="org.scalastyle.scalariform.SimplifyBooleanExpressionChecker" enabled="true"></check>
74+
<check level="error" class="org.scalastyle.scalariform.IfBraceChecker" enabled="true">
75+
<parameters>
76+
<parameter name="singleLineAllowed"><![CDATA[true]]></parameter>
77+
<parameter name="doubleLineAllowed"><![CDATA[false]]></parameter>
78+
</parameters>
79+
</check>
80+
<check level="error" class="org.scalastyle.scalariform.MethodLengthChecker" enabled="true">
81+
<parameters>
82+
<parameter name="maxLength"><![CDATA[50]]></parameter>
83+
</parameters>
84+
</check>
85+
<check level="error" class="org.scalastyle.scalariform.MethodNamesChecker" enabled="true">
86+
<parameters>
87+
<parameter name="regex"><![CDATA[^[a-z][A-Za-z0-9]*$]]></parameter>
88+
</parameters>
89+
</check>
90+
<check level="error" class="org.scalastyle.scalariform.NumberOfMethodsInTypeChecker" enabled="true">
91+
<parameters>
92+
<parameter name="maxMethods"><![CDATA[30]]></parameter>
93+
</parameters>
94+
</check>
95+
<check level="error" class="org.scalastyle.scalariform.PublicMethodsHaveTypeChecker" enabled="false"></check>
96+
<check level="error" class="org.scalastyle.file.NewLineAtEofChecker" enabled="true"></check>
97+
<check level="error" class="org.scalastyle.file.NoNewLineAtEofChecker" enabled="false"></check>
98+
</scalastyle>

‎src/main/scala/com/github/caiiiycuk/pg2sqlite/Boot.scala

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package com.github.caiiiycuk.pg2sqlite
2+
3+
import com.github.caiiiycuk.pg2sqlite.command.CommandException
4+
import com.github.caiiiycuk.pg2sqlite.iterator.LineIterator
5+
import com.github.caiiiycuk.pg2sqlite.values.ValueParseException
6+
7+
import ch.qos.logback.classic.Level
8+
9+
object Boot extends App with Log {
10+
11+
val root = org.slf4j.LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger]
12+
root.setLevel(Level.INFO)
13+
14+
val config = Config.parse(args)
15+
import config._
16+
17+
val size = pgdump.length()
18+
val connection = Connection.sqlite(sqlite)
19+
val iterator = LineIterator(pgdump)
20+
val loggedIterator = LoggedIterator(iterator, () => 100.0 * iterator.readed / size)
21+
val dumpInserter = new DumpInserter(connection)
22+
23+
log.info(s"'$pgdump' (${toMb(size)} Mb) -> '$sqlite'")
24+
25+
val success = try {
26+
dumpInserter.insert(loggedIterator)
27+
true
28+
} catch {
29+
case e: CommandException =>
30+
log.error(e.getMessage)
31+
false
32+
case e: ValueParseException =>
33+
log.error(e.getMessage)
34+
false
35+
case e: Throwable =>
36+
log.error(e.getMessage, e)
37+
false
38+
}
39+
40+
iterator.close
41+
connection.close
42+
43+
if (success) {
44+
log.info("Well done...")
45+
} else {
46+
log.error("Task failed...")
47+
}
48+
}

‎src/main/scala/com/github/caiiiycuk/pg2sqlite/Config.scala

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package com.github.caiiiycuk.pg2sqlite
2+
3+
import java.io.File
4+
5+
case class Config(pgdump: File = new File("dump"), sqlite: File = new File("db"), force: Boolean = false)
6+
7+
object Config extends Log {
8+
private val parser = new scopt.OptionParser[Config]("postgresql-to-sqlite") {
9+
head("postgresql-to-sqlite")
10+
11+
opt[File]('d', "dump") required () valueName ("<dump file>") action { (v, c) =>
12+
c.copy(pgdump = v)
13+
} text ("postgresql dump generated by pg_dump")
14+
15+
opt[File]('o', "out") required () valueName ("<sqlite3 database>") action { (v, c) =>
16+
c.copy(sqlite = v)
17+
} text ("sqlite3 database to create")
18+
19+
opt[Boolean]('f', "force") optional () valueName ("<true|false>") action { (v, c) =>
20+
c.copy(force = v)
21+
} text ("recreate database if exists")
22+
23+
checkConfig { c =>
24+
import c._
25+
26+
if (!pgdump.exists()) {
27+
failure(s"Dump '${pgdump}' does not exists")
28+
} else if (sqlite.exists()) {
29+
if (force) {
30+
sqlite.delete()
31+
success
32+
} else {
33+
failure(s"Database '${sqlite}' already exists")
34+
}
35+
} else {
36+
success
37+
}
38+
}
39+
}
40+
41+
def parse(args: Array[String]) = {
42+
parser.parse(args, Config()) match {
43+
case Some(config) =>
44+
Option(System.getenv("SQLITE_TMPDIR")) match {
45+
case None =>
46+
log.warn("You should set SQLITE_TMPDIR environment variable to control where sqlite stores temp files")
47+
case _ =>
48+
}
49+
50+
config
51+
case _ =>
52+
System.exit(1)
53+
???
54+
}
55+
}
56+
57+
}

‎src/main/scala/com/github/caiiiycuk/pg2sqlite/Connection.scala

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package com.github.caiiiycuk.pg2sqlite
2+
3+
import java.sql.DriverManager
4+
import java.sql.Statement
5+
import java.sql.PreparedStatement
6+
import scala.collection.mutable.ListBuffer
7+
import java.sql.ResultSet
8+
import scala.annotation.tailrec
9+
import java.io.File
10+
11+
trait ConnectionHolder {
12+
def makeConnection: java.sql.Connection
13+
def db: String
14+
}
15+
16+
object Connection {
17+
final val FETCH_SIZE = 8192
18+
final val MAX_VARIABLE_NUMBER = 999
19+
20+
def sqlite(dbFile: File) = {
21+
val connectionHolder = new ConnectionHolder {
22+
override def makeConnection: java.sql.Connection = {
23+
implicit val connection = DriverManager.getConnection(s"jdbc:sqlite:$dbFile")
24+
25+
connection.setAutoCommit(true)
26+
sqlitePragmas()
27+
28+
connection.setAutoCommit(false)
29+
connection
30+
}
31+
32+
override def db = dbFile.toString
33+
}
34+
35+
new Connection(connectionHolder)
36+
}
37+
38+
private def sqlitePragmas()(implicit connection: java.sql.Connection) = {
39+
val statment = connection.createStatement()
40+
statment.executeUpdate("PRAGMA synchronous = OFF")
41+
statment.executeUpdate("PRAGMA journal_mode = OFF")
42+
statment.executeUpdate("PRAGMA threads = 64")
43+
statment.executeUpdate("PRAGMA max_page_count = 2147483646")
44+
statment.executeUpdate("PRAGMA cache_size = 65536")
45+
statment.executeUpdate("PRAGMA cache_spill = true")
46+
statment.close
47+
}
48+
}
49+
50+
class Connection(connectionHolder: ConnectionHolder) {
51+
52+
import Connection._
53+
54+
final val MAX_VARIABLE_NUMBER = Connection.MAX_VARIABLE_NUMBER
55+
56+
lazy val connection = connectionHolder.makeConnection
57+
58+
lazy val db = connectionHolder.db
59+
60+
def withStatement[T](block: (Statement) => T): T = {
61+
val statement = connection.createStatement()
62+
val t = block(statement)
63+
statement.close
64+
t
65+
}
66+
67+
def withPreparedStatement[T](sql: String, keepAlive: Boolean = false)(block: (PreparedStatement) => T): T = {
68+
val statement = connection.prepareStatement(sql)
69+
statement.setFetchSize(FETCH_SIZE)
70+
71+
val t = block(statement)
72+
if (!keepAlive) statement.close
73+
t
74+
}
75+
76+
def close = {
77+
connection.commit
78+
connection.close
79+
}
80+
81+
def execute(sql: String) = {
82+
withStatement { statement =>
83+
statement.executeUpdate(sql)
84+
}
85+
}
86+
87+
}

‎src/main/scala/com/github/caiiiycuk/pg2sqlite/DumpInserter.scala

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package com.github.caiiiycuk.pg2sqlite
2+
3+
import scala.annotation.tailrec
4+
import com.github.caiiiycuk.pg2sqlite.command._
5+
import com.github.caiiiycuk.pg2sqlite.iterator.Line
6+
import com.github.caiiiycuk.pg2sqlite.schema.Schema
7+
8+
object DumpInserter {
9+
val COMMANDS = List(CreateTable, Copy, CreateIndex)
10+
}
11+
12+
class DumpInserter(connection: Connection) {
13+
14+
import DumpInserter._
15+
16+
implicit val schema = new Schema()
17+
18+
@tailrec
19+
final def insert(iterator: Iterator[Line]): Unit = {
20+
if (iterator.hasNext) {
21+
val head = iterator.next()
22+
val fullIterator = Iterator(head) ++ iterator
23+
24+
COMMANDS.find(_.matchHead(head)).map { command =>
25+
command.apply(connection, fullIterator)
26+
}
27+
28+
insert(iterator)
29+
}
30+
}
31+
32+
}

0 commit comments

Comments
 (0)