Skip to content
This repository was archived by the owner on Oct 12, 2023. It is now read-only.

Commit 55e999b

Browse files
author
Zeqi Cui
committed
Added sample scripts for use cases
+ Fixed failing spec for bulk copy metadata
1 parent f0bd19e commit 55e999b

File tree

4 files changed

+129
-3
lines changed

4 files changed

+129
-3
lines changed

samples/scripts/BulkCopySample.scala

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// Import libraries
2+
import com.microsoft.azure.sqldb.spark.bulkcopy.BulkCopyMetadata
3+
import com.microsoft.azure.sqldb.spark.config.Config
4+
import com.microsoft.azure.sqldb.spark.connect._
5+
6+
val url = "[Enter your url here]"
7+
val databaseName = "[Enter your database name here]"
8+
val dbTable = "[Enter your database table here]"
9+
10+
val user = "[Enter your username here]"
11+
val password = "[Enter your password here]"
12+
13+
// Acquire data to be written.
14+
// df could be aquired in any way.
15+
val localTable = "[Enter your local persisted table here]"
16+
val df = spark.sql(s"SELECT * FROM $localTable")
17+
18+
val writeConfig = Config(Map(
19+
"url" -> url,
20+
"databaseName" -> databaseName,
21+
"dbTable" -> dbTable,
22+
"user" -> user,
23+
"password" -> password,
24+
"connectTimeout" -> "5",
25+
"bulkCopyBatchSize" -> "100000",
26+
"bulkCopyTableLock" -> "true",
27+
"bulkCopyTimeout" -> "600"
28+
))
29+
30+
df.bulkCopyToSqlDB(writeConfig)
31+
32+
/**
33+
For better performance, specify the column metadata of the table
34+
35+
var bulkCopyMetadata = new BulkCopyMetadata
36+
bulkCopyMetadata.addColumnMetadata(1, "Title", java.sql.Types.NVARCHAR, 128, 0)
37+
bulkCopyMetadata.addColumnMetadata(2, "FirstName", java.sql.Types.NVARCHAR, 128, 0)
38+
bulkCopyMetadata.addColumnMetadata(3, "MiddleName", java.sql.Types.NVARCHAR, 128, 0)
39+
bulkCopyMetadata.addColumnMetadata(4, "LastName", java.sql.Types.NVARCHAR, 128, 0)
40+
..........
41+
42+
df.bulkCopyToSqlDB(writeConfig, bulkCopyMetadata)
43+
**/

samples/scripts/ReadSample.scala

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Import libraries
2+
import com.microsoft.azure.sqldb.spark.bulkcopy.BulkCopyMetadata
3+
import com.microsoft.azure.sqldb.spark.config.Config
4+
import com.microsoft.azure.sqldb.spark.connect._
5+
6+
val url = "[Enter your url here]"
7+
val databaseName = "[Enter your database name here]"
8+
val dbTable = "[Enter your database table here]"
9+
10+
val user = "[Enter your username here]"
11+
val password = "[Enter your password here]"
12+
13+
// READ FROM CONFIG
14+
val readConfig = Config(Map(
15+
"url" -> url,
16+
"databaseName" -> databaseName,
17+
"user" -> user,
18+
"password" -> password,
19+
"connectTimeout" -> "5",
20+
"queryTimeout" -> "5",
21+
"dbTable" -> dbTable
22+
))
23+
24+
val df = sqlContext.read.sqlDB(readConfig)
25+
println("Total rows: " + df.count)
26+
df.show()
27+
28+
// TRADITIONAL SYNTAX
29+
import java.util.Properties
30+
31+
val properties = new Properties()
32+
properties.put("databaseName", databaseName)
33+
properties.put("user", user)
34+
properties.put("password", password)
35+
properties.put("connectTimeout", "5")
36+
properties.put("queryTimeout", "5")
37+
38+
val df = sqlContext.read.sqlDB(url, dbTable, properties)
39+
println("Total rows: " + df.count)
40+
df.show()

samples/scripts/WriteSample.scala

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Import libraries
2+
import com.microsoft.azure.sqldb.spark.config.Config
3+
import com.microsoft.azure.sqldb.spark.connect._
4+
5+
val url = "[Enter your url here]"
6+
val databaseName = "[Enter your database name here]"
7+
val dbTable = "[Enter your database table here]"
8+
9+
val user = "[Enter your username here]"
10+
val password = "[Enter your password here]"
11+
12+
// Acquire data to be written.
13+
// df could be aquired in any way.
14+
val localTable = "[Enter your local persisted table here]"
15+
val df = spark.sql(s"SELECT * FROM $localTable")
16+
17+
// WRITE FROM CONFIG
18+
val writeConfig = Config(Map(
19+
"url" -> url,
20+
"databaseName" -> databaseName,
21+
"dbTable" -> dbTable,
22+
"user" -> user,
23+
"password" -> password,
24+
"connectTimeout" -> "5",
25+
"queryTimeout" -> "5"
26+
))
27+
28+
df.write.mode(SaveMode.Append).sqlDB(writeConfig)
29+
30+
/** TRADITIONAL SYNTAX
31+
32+
import java.util.Properties
33+
34+
val properties = new Properties()
35+
36+
properties.put("databaseName", databaseName)
37+
properties.put("user", user)
38+
properties.put("password", password)
39+
properties.put("connectTimeout", "5")
40+
properties.put("queryTimeout", "5")
41+
42+
df.write.mode(SaveMode.Append).sqlDB(url, dbTable, properties)
43+
44+
**/

src/test/java/com/microsoft/azure/sqldb/spark/bulkcopy/BulkCopyMetadataSpec.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,16 +47,15 @@ public void constructorTest(){
4747
@Test
4848
public void addColumnMetadataTest() {
4949
String columnName = "testColumn";
50-
int column = 1;
5150
int jdbcType = Types.DOUBLE;
5251
int precision = 15;
5352
int scale = 5;
5453
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss");
5554

56-
bulkCopyMetadata.addColumnMetadata(column, columnName, jdbcType, precision, scale);
55+
bulkCopyMetadata.addColumnMetadata(1, columnName, jdbcType, precision, scale);
5756
assertEquals(1, bulkCopyMetadata.getMetadata().size());
5857

59-
bulkCopyMetadata.addColumnMetadata(column, columnName, jdbcType, precision, scale, dateTimeFormatter);
58+
bulkCopyMetadata.addColumnMetadata(2, columnName, jdbcType, precision, scale, dateTimeFormatter);
6059
assertEquals(2, bulkCopyMetadata.getMetadata().size());
6160
}
6261
}

0 commit comments

Comments
 (0)