Skip to content

Commit

Permalink
Use safer 'TRY_CONVERT' with the explicit format reference number for…
Browse files Browse the repository at this point in the history 
… MS SQL Server.

Rather than rely on non-deterministic ISDATE().
  • Loading branch information
@yruslan
yruslan committed Dec 18, 2023
1 parent 34b6002 commit 3773b2b
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 25 deletions.
13 changes: 8 additions & 5 deletions pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMicrosoft.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ class SqlGeneratorMicrosoft(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlCo
private val dateFormatterApp = DateTimeFormatter.ofPattern(sqlConfig.dateFormatApp)
private val isIso = sqlConfig.dateFormatApp.toLowerCase.startsWith("yyyy-mm-dd")

// 23 is "yyyy-MM-dd", see https://www.mssqltips.com/sqlservertip/1145/date-and-time-conversions-using-sql-server/
private val isoFormatMsSqlRef = 23

override val beginEndEscapeChars: (Char, Char) = ('[', ']')

override def getDtable(sql: String): String = {
Expand Down Expand Up @@ -56,9 +59,9 @@ class SqlGeneratorMicrosoft(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlCo
val dateEndLit = getDateLiteral(dateEnd)

val infoDateColumnAdjusted = if (sqlConfig.infoDateType == SqlColumnType.DATETIME) {
s"CONVERT(DATE, $infoDateColumn)"
s"CONVERT(DATE, $infoDateColumn, $isoFormatMsSqlRef)"
} else if (sqlConfig.infoDateType == SqlColumnType.STRING && isIso) {
s"(CASE WHEN ISDATE($infoDateColumn) = 1 THEN CONVERT(DATE, $infoDateColumn) ELSE NULL END)"
s"TRY_CONVERT(DATE, $infoDateColumn, $isoFormatMsSqlRef)"
} else {
infoDateColumn
}
Expand All @@ -74,14 +77,14 @@ class SqlGeneratorMicrosoft(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlCo
sqlConfig.infoDateType match {
case SqlColumnType.DATE =>
val dateStr = DateTimeFormatter.ISO_LOCAL_DATE.format(date)
s"CONVERT(DATE, '$dateStr')"
s"CONVERT(DATE, '$dateStr', $isoFormatMsSqlRef)"
case SqlColumnType.DATETIME =>
val dateStr = DateTimeFormatter.ISO_LOCAL_DATE.format(date)
s"CONVERT(DATE, '$dateStr')"
s"CONVERT(DATE, '$dateStr', $isoFormatMsSqlRef)"
case SqlColumnType.STRING =>
if (isIso) {
val dateStr = DateTimeFormatter.ISO_LOCAL_DATE.format(date)
s"CONVERT(DATE, '$dateStr')"
s"CONVERT(DATE, '$dateStr', $isoFormatMsSqlRef)"
} else {
val dateStr = dateFormatterApp.format(date)
s"'$dateStr'"
Expand Down
40 changes: 20 additions & 20 deletions pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,24 +204,24 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture {
"generate ranged count queries" when {
"date is in DATE format" in {
assert(genDate.getCountQuery("A", date1, date1) ==
"SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17')")
"SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23)")
assert(genDate.getCountQuery("A", date1, date2) ==
"SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D >= CONVERT(DATE, '2020-08-17') AND D <= CONVERT(DATE, '2020-08-30')")
"SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D >= CONVERT(DATE, '2020-08-17', 23) AND D <= CONVERT(DATE, '2020-08-30', 23)")
}

"date is in DATETIME format" in {
assert(genDateTime.getCountQuery("A", date1, date1) ==
"SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D) = CONVERT(DATE, '2020-08-17')")
"SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
assert(genDateTime.getCountQuery("A", date1, date2) ==
"SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D) >= CONVERT(DATE, '2020-08-17') AND CONVERT(DATE, D) <= CONVERT(DATE, '2020-08-30')")
"SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) >= CONVERT(DATE, '2020-08-17', 23) AND CONVERT(DATE, D, 23) <= CONVERT(DATE, '2020-08-30', 23)")
}

"date is in STRING ISO format" in {
assert(genStr.getCountQuery("A", date1, date1) ==
"SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE (CASE WHEN ISDATE(D) = 1 THEN CONVERT(DATE, D) ELSE NULL END) = CONVERT(DATE, '2020-08-17')")
"SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE TRY_CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
assert(genStr.getCountQuery("A", date1, date2) ==
"SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE (CASE WHEN ISDATE(D) = 1 THEN CONVERT(DATE, D) ELSE NULL END) >= CONVERT(DATE, '2020-08-17') " +
"AND (CASE WHEN ISDATE(D) = 1 THEN CONVERT(DATE, D) ELSE NULL END) <= CONVERT(DATE, '2020-08-30')")
"SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE TRY_CONVERT(DATE, D, 23) >= CONVERT(DATE, '2020-08-17', 23) " +
"AND TRY_CONVERT(DATE, D, 23) <= CONVERT(DATE, '2020-08-30', 23)")
}

"date is in STRING non ISO format" in {
Expand All @@ -240,40 +240,40 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture {

"the table name and column name need to be escaped" in {
assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
"SELECT COUNT(*) AS CNT FROM [Input Table] WITH (NOLOCK) WHERE [Info date] = CONVERT(DATE, '2020-08-17')")
"SELECT COUNT(*) AS CNT FROM [Input Table] WITH (NOLOCK) WHERE [Info date] = CONVERT(DATE, '2020-08-17', 23)")
assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
"SELECT COUNT(*) AS CNT FROM [Input Table] WITH (NOLOCK) WHERE [Info date] >= CONVERT(DATE, '2020-08-17') AND [Info date] <= CONVERT(DATE, '2020-08-30')")
"SELECT COUNT(*) AS CNT FROM [Input Table] WITH (NOLOCK) WHERE [Info date] >= CONVERT(DATE, '2020-08-17', 23) AND [Info date] <= CONVERT(DATE, '2020-08-30', 23)")
}

"the table name and column name already escaped" in {
assert(genEscaped2.getCountQuery("Input Table", date1, date1) ==
"SELECT COUNT(*) AS CNT FROM [Input Table] WITH (NOLOCK) WHERE [Info date] = CONVERT(DATE, '2020-08-17')")
"SELECT COUNT(*) AS CNT FROM [Input Table] WITH (NOLOCK) WHERE [Info date] = CONVERT(DATE, '2020-08-17', 23)")
assert(genEscaped2.getCountQuery("Input Table", date1, date2) ==
"SELECT COUNT(*) AS CNT FROM [Input Table] WITH (NOLOCK) WHERE [Info date] >= CONVERT(DATE, '2020-08-17') AND [Info date] <= CONVERT(DATE, '2020-08-30')")
"SELECT COUNT(*) AS CNT FROM [Input Table] WITH (NOLOCK) WHERE [Info date] >= CONVERT(DATE, '2020-08-17', 23) AND [Info date] <= CONVERT(DATE, '2020-08-30', 23)")
}
}

"generate ranged data queries" when {
"date is in DATE format" in {
assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
"SELECT * FROM A WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17')")
"SELECT * FROM A WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23)")
assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
"SELECT * FROM A WITH (NOLOCK) WHERE D >= CONVERT(DATE, '2020-08-17') AND D <= CONVERT(DATE, '2020-08-30')")
"SELECT * FROM A WITH (NOLOCK) WHERE D >= CONVERT(DATE, '2020-08-17', 23) AND D <= CONVERT(DATE, '2020-08-30', 23)")
}

"date is in DATETIME format" in {
assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
"SELECT * FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D) = CONVERT(DATE, '2020-08-17')")
"SELECT * FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
"SELECT * FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D) >= CONVERT(DATE, '2020-08-17') AND CONVERT(DATE, D) <= CONVERT(DATE, '2020-08-30')")
"SELECT * FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) >= CONVERT(DATE, '2020-08-17', 23) AND CONVERT(DATE, D, 23) <= CONVERT(DATE, '2020-08-30', 23)")
}

"date is in STRING ISO format" in {
assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
"SELECT * FROM A WITH (NOLOCK) WHERE (CASE WHEN ISDATE(D) = 1 THEN CONVERT(DATE, D) ELSE NULL END) = CONVERT(DATE, '2020-08-17')")
"SELECT * FROM A WITH (NOLOCK) WHERE TRY_CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
"SELECT * FROM A WITH (NOLOCK) WHERE (CASE WHEN ISDATE(D) = 1 THEN CONVERT(DATE, D) ELSE NULL END) >= CONVERT(DATE, '2020-08-17') " +
"AND (CASE WHEN ISDATE(D) = 1 THEN CONVERT(DATE, D) ELSE NULL END) <= CONVERT(DATE, '2020-08-30')")
"SELECT * FROM A WITH (NOLOCK) WHERE TRY_CONVERT(DATE, D, 23) >= CONVERT(DATE, '2020-08-17', 23) " +
"AND TRY_CONVERT(DATE, D, 23) <= CONVERT(DATE, '2020-08-30', 23)")
}

"date is in STRING non-ISO format" in {
Expand All @@ -292,9 +292,9 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture {

"with limit records" in {
assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
"SELECT TOP 100 * FROM A WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17')")
"SELECT TOP 100 * FROM A WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23)")
assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
"SELECT TOP 100 * FROM A WITH (NOLOCK) WHERE D >= CONVERT(DATE, '2020-08-17') AND D <= CONVERT(DATE, '2020-08-30')")
"SELECT TOP 100 * FROM A WITH (NOLOCK) WHERE D >= CONVERT(DATE, '2020-08-17', 23) AND D <= CONVERT(DATE, '2020-08-30', 23)")
}
}

Expand Down

0 comments on commit 3773b2b

Please sign in to comment.