Skip to content

Commit a6eb3a7

Browse files
kmacmcfarlanekmacmcfarlane
committed
fixed: issues with postgresql wrapper and database writer
changed: no longer use an ID column (it was causing issues)
1 parent 400809a commit a6eb3a7

File tree

11 files changed

+1272
-130
lines changed

11 files changed

+1272
-130
lines changed

Diff for: ‎.idea/go.imports.xml

+6
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎.idea/workspace.xml

+185-87
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎Makefile

+4-1
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,7 @@ build_mocks:
1818
@mockery -all -dir postgresql/ -output mock -case=underscore
1919

2020
test: build_mocks
21-
ginkgo -r ./
21+
ginkgo -r ./
22+
23+
test_watch: build_mocks
24+
ginkgo watch -r ./

Diff for: ‎bin/csv-2-postgresql

2.57 MB
Binary file not shown.

Diff for: ‎csv-2-postgresql/database_writer.go

-2
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ import "github.com/kmacmcfarlane/csv-2-postgresql-go/schema"
55
// DatabaseWriter defines a generic interface to a database
66
type DatabaseWriter interface {
77

8-
// CreateDatabase creates a database if it does not exist
9-
CreateDatabase(name string) (err error)
108
// CreateTable instantiates a table compatible with the given schema
119
CreateTable(name string, schema schema.Schema) error
1210
// Insert adds a record to the table

Diff for: ‎csv-2-postgresql/main.go

+19-23
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ package main
22

33
import (
44
"database/sql"
5-
"flag"
65
"fmt"
76
"github.com/iancoleman/strcase"
87
"github.com/kmacmcfarlane/csv-2-postgresql-go/csv"
98
"github.com/kmacmcfarlane/csv-2-postgresql-go/postgresql"
9+
_ "github.com/lib/pq"
1010
"io"
1111
"os"
1212
"path/filepath"
@@ -16,25 +16,15 @@ import (
1616
func main() {
1717

1818
// Parse arguments
19-
args := flag.Args()
19+
args := os.Args[1:]
2020

21-
if len(args) != 1 || len(args) != 2 {
21+
if len(args) != 2 {
2222
usage()
2323
os.Exit(1)
2424
}
2525

2626
inputFile := args[0]
27-
28-
29-
var databaseName string
30-
if len(args) == 2 {
31-
databaseName = args[1]
32-
} else {
33-
databaseName = "test"
34-
}
35-
36-
// Connect to DB
37-
connectionString := fmt.Sprintf("user=postgres dbname=%s sslmode=verify-full", databaseName)
27+
connectionString := args[1]
3828

3929
db, err := sql.Open("postgres", connectionString)
4030

@@ -46,7 +36,7 @@ func main() {
4636

4737
sqlWrapper := postgresql.NewSQLWrapper(db)
4838

49-
dbWriter := postgresql.NewWriter(sqlWrapper)
39+
var dbWriter DatabaseWriter = postgresql.NewWriter(sqlWrapper)
5040

5141
// Parse schema of input
5242
file, err := os.Open(inputFile)
@@ -65,6 +55,7 @@ func main() {
6555

6656
// Create table
6757
_, filenameAlone := filepath.Split(file.Name())
58+
filenameAlone = strings.Replace(filenameAlone, ".csv", "", 1)
6859
tableName := strcase.ToSnake(filenameAlone)
6960

7061
err = dbWriter.CreateTable(tableName, schema)
@@ -73,39 +64,44 @@ func main() {
7364
panic(fmt.Sprintf("error creating db table: %s", err.Error()))
7465
}
7566

76-
fmt.Printf("created table '%s' in database %s", tableName, databaseName)
67+
fmt.Printf("created table '%s'\n", tableName)
7768

7869
// Insert data to database
79-
for {
70+
count := 0
71+
for i := 0; true; i++ {
8072

8173
record, err := parser.Read()
8274

8375
if nil != err {
8476

8577
if io.EOF == err {
78+
79+
count = i
80+
8681
break
8782
}
8883

89-
fmt.Printf("error parsing record: %s", strings.Join(record, ", "))
90-
fmt.Printf("error: %s", err.Error())
84+
fmt.Printf("error parsing record: %s\n", strings.Join(record, ", "))
85+
fmt.Printf("error: %s\n", err.Error())
9186
}
9287

9388
err = dbWriter.Insert(record, schema, tableName)
9489

9590
if nil != err {
9691

97-
fmt.Printf("error inserting record: %s", strings.Join(record, ", "))
98-
fmt.Printf("error: %s", err.Error())
92+
fmt.Printf("error inserting record: %s\n", strings.Join(record, ", "))
93+
fmt.Printf("error: %s\n", err.Error())
9994
}
10095
}
10196

102-
println("import ")
97+
fmt.Printf("processed %d records\n", count)
10398

10499
os.Exit(0)
105100
}
106101

107102
func usage(){
108103

109104
println("Usage:")
110-
println("csv-2-postgresql input_file.csv [database_name]")
105+
println("make")
106+
println("./bin/csv-2-postgresql ./input_file.csv postgres://postgres:hello@localhost/test?sslmode=disable")
111107
}

Diff for: ‎postgresql/postgresql_test.go

+39-14
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package postgresql_test
22

33
import (
4+
"fmt"
45
"github.com/kmacmcfarlane/csv-2-postgresql-go/mock"
56
"github.com/kmacmcfarlane/csv-2-postgresql-go/postgresql"
67
"github.com/kmacmcfarlane/csv-2-postgresql-go/schema"
@@ -29,13 +30,16 @@ var _ = Describe("Postgresql Database Adapter", func() {
2930
Context("Small integer column", func(){
3031
It("Creates a table with a single SMALLINT column", func(){
3132

32-
sql.On("Exec",
33+
sql.
34+
On("Exec", fmt.Sprintf(`DROP TABLE IF EXISTS "tablename";`)).
35+
Return(new(mocks.Result), nil)
36+
37+
sql.
38+
On("Exec",
3339
`CREATE TABLE "tablename" (
34-
_id UUID GENERATED ALWAYS AS IDENTITY,
3540
"total" SMALLINT NOT NULL
3641
);`).
37-
Return(new(mocks.Result), nil).
38-
Times(1)
42+
Return(new(mocks.Result), nil)
3943

4044
schema := schema.Schema{
4145
Columns: []schema.Column{
@@ -54,9 +58,12 @@ _id UUID GENERATED ALWAYS AS IDENTITY,
5458
Context("int16 integer column", func(){
5559
It("Creates a table with a single SMALLINT column", func(){
5660

61+
sql.
62+
On("Exec", fmt.Sprintf(`DROP TABLE IF EXISTS "tablename";`)).
63+
Return(new(mocks.Result), nil)
64+
5765
sql.On("Exec",
5866
`CREATE TABLE "tablename" (
59-
_id UUID GENERATED ALWAYS AS IDENTITY,
6067
"total" SMALLINT NOT NULL
6168
);`).
6269
Return(new(mocks.Result), nil).
@@ -79,9 +86,12 @@ _id UUID GENERATED ALWAYS AS IDENTITY,
7986
Context("int32 integer column", func(){
8087
It("Creates a table with a single INTEGER column", func(){
8188

89+
sql.
90+
On("Exec", fmt.Sprintf(`DROP TABLE IF EXISTS "tablename";`)).
91+
Return(new(mocks.Result), nil)
92+
8293
sql.On("Exec",
8394
`CREATE TABLE "tablename" (
84-
_id UUID GENERATED ALWAYS AS IDENTITY,
8595
"total" INTEGER NOT NULL
8696
);`).
8797
Return(new(mocks.Result), nil).
@@ -104,9 +114,12 @@ _id UUID GENERATED ALWAYS AS IDENTITY,
104114
Context("int64 integer column", func(){
105115
It("Creates a table with a single BIGINT column", func(){
106116

117+
sql.
118+
On("Exec", fmt.Sprintf(`DROP TABLE IF EXISTS "tablename";`)).
119+
Return(new(mocks.Result), nil)
120+
107121
sql.On("Exec",
108122
`CREATE TABLE "tablename" (
109-
_id UUID GENERATED ALWAYS AS IDENTITY,
110123
"total" BIGINT NOT NULL
111124
);`).
112125
Return(new(mocks.Result), nil).
@@ -129,9 +142,12 @@ _id UUID GENERATED ALWAYS AS IDENTITY,
129142
Context("float32 column", func(){
130143
It("Creates a table with a single FLOAT column", func(){
131144

145+
sql.
146+
On("Exec", fmt.Sprintf(`DROP TABLE IF EXISTS "tablename";`)).
147+
Return(new(mocks.Result), nil)
148+
132149
sql.On("Exec",
133150
`CREATE TABLE "tablename" (
134-
_id UUID GENERATED ALWAYS AS IDENTITY,
135151
"total" FLOAT NOT NULL
136152
);`).
137153
Return(new(mocks.Result), nil).
@@ -154,9 +170,12 @@ _id UUID GENERATED ALWAYS AS IDENTITY,
154170
Context("float64 column", func(){
155171
It("Creates a table with a single DOUBLE column", func(){
156172

173+
sql.
174+
On("Exec", fmt.Sprintf(`DROP TABLE IF EXISTS "tablename";`)).
175+
Return(new(mocks.Result), nil)
176+
157177
sql.On("Exec",
158178
`CREATE TABLE "tablename" (
159-
_id UUID GENERATED ALWAYS AS IDENTITY,
160179
"total" DOUBLE NOT NULL
161180
);`).
162181
Return(new(mocks.Result), nil).
@@ -179,9 +198,12 @@ _id UUID GENERATED ALWAYS AS IDENTITY,
179198
Context("string column", func(){
180199
It("Creates a table with a single TEXT column", func(){
181200

201+
sql.
202+
On("Exec", fmt.Sprintf(`DROP TABLE IF EXISTS "tablename";`)).
203+
Return(new(mocks.Result), nil)
204+
182205
sql.On("Exec",
183206
`CREATE TABLE "tablename" (
184-
_id UUID GENERATED ALWAYS AS IDENTITY,
185207
"comments" TEXT NOT NULL
186208
);`).
187209
Return(new(mocks.Result), nil).
@@ -204,9 +226,12 @@ _id UUID GENERATED ALWAYS AS IDENTITY,
204226
Context("Multiple columns", func(){
205227
It("Creates a table with INTEGER, FLOAT, and TEXT columns", func(){
206228

229+
sql.
230+
On("Exec", fmt.Sprintf(`DROP TABLE IF EXISTS "tablename";`)).
231+
Return(new(mocks.Result), nil)
232+
207233
sql.On("Exec",
208234
`CREATE TABLE "tablename" (
209-
_id UUID GENERATED ALWAYS AS IDENTITY,
210235
"count" INTEGER NOT NULL,
211236
"total" FLOAT NOT NULL,
212237
"comments" TEXT NOT NULL
@@ -241,7 +266,7 @@ _id UUID GENERATED ALWAYS AS IDENTITY,
241266
It("Inserts an integer value", func(){
242267

243268
sql.On("Exec",
244-
`INSERT INTO "tablename" VALUES ($1);`, []string{"13"}).
269+
`INSERT INTO "tablename" VALUES ($1);`, "13").
245270
Return(new(mocks.Result), nil).
246271
Times(1)
247272

@@ -263,7 +288,7 @@ _id UUID GENERATED ALWAYS AS IDENTITY,
263288
It("Inserts a string value", func(){
264289

265290
sql.On("Exec",
266-
`INSERT INTO "tablename" VALUES ($1);`, []string{"hello world"}).
291+
`INSERT INTO "tablename" VALUES ($1);`, "hello world").
267292
Return(new(mocks.Result), nil).
268293
Times(1)
269294

@@ -285,7 +310,7 @@ _id UUID GENERATED ALWAYS AS IDENTITY,
285310
It("Inserts an int, float, and string value", func(){
286311

287312
sql.On("Exec",
288-
`INSERT INTO "tablename" VALUES ($1, $2, $3);`, []string{"13", "321.123", "hello world"}).
313+
`INSERT INTO "tablename" VALUES ($1, $2, $3);`, "13", "321.123", "hello world").
289314
Return(new(mocks.Result), nil).
290315
Times(1)
291316

Diff for: ‎postgresql/sql_wrapper.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ func NewSQLWrapper(inner *sql.DB) SQLWrapper {
2222
}
2323

2424
func (s SQLWrapper) Exec(query string, args ...interface{}) (Result, error) {
25-
return s.inner.Exec(query, args)
25+
return s.inner.Exec(query, args...)
2626
}
2727

2828
func (s SQLWrapper) Close() error {

Diff for: ‎postgresql/writer.go

+13-2
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,13 @@ func (w Writer) CreateTable(tableName string, schema schema.Schema) (err error)
2626
return errors.New("invalid schema: missing columns")
2727
}
2828

29+
// Remove existing table
30+
_, err = w.db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS "%s";`, tableName))
31+
32+
if nil != err {
33+
return err
34+
}
35+
2936
// Define the schema
3037
columnDefinitions := make([]string, len(schema.Columns))
3138

@@ -41,7 +48,6 @@ func (w Writer) CreateTable(tableName string, schema schema.Schema) (err error)
4148
// Execute the statement
4249
statementTemplate :=
4350
`CREATE TABLE "%s" (
44-
_id UUID GENERATED ALWAYS AS IDENTITY,
4551
%s
4652
);`
4753

@@ -76,7 +82,12 @@ func (w Writer) Insert(values []string, schema schema.Schema, tableName string)
7682

7783
statement := fmt.Sprintf(`INSERT INTO "%s" VALUES (%s);`, tableName, sb.String())
7884

79-
_, err = w.db.Exec(statement, values)
85+
var valuesGeneric = make([]interface{}, len(values))
86+
for i, value := range values {
87+
valuesGeneric[i] = value
88+
}
89+
90+
_, err = w.db.Exec(statement, valuesGeneric...)
8091

8192
if nil != err {
8293
return err

0 commit comments

Comments
 (0)