From 383290627f2f17be2ad7313d568ebf811ab5c153 Mon Sep 17 00:00:00 2001 From: Taylor Curran Date: Mon, 27 Jan 2025 09:17:48 -0800 Subject: [PATCH] #1020: `earliest`/`latest` not supported with `where` command (#1021) * Add earliest/latest to `conditionFunctionBase`. Signed-off-by: currantw * Update doc and integ tests Signed-off-by: currantw * Update/fix doc tests Signed-off-by: currantw * Rename variable Signed-off-by: currantw * scalafmtAll Signed-off-by: currantw * Fix failing integ test Signed-off-by: currantw * More scalafmt. Grrr... Signed-off-by: currantw --------- Signed-off-by: currantw --- docs/ppl-lang/functions/ppl-datetime.md | 99 +++----- integ-test/script/data/relative_datetime.json | 10 + .../data/relative_datetime.mapping.json | 12 + .../flint/spark/FlintSparkSuite.scala | 20 ++ ...arkPPLBuiltInDateTimeFunctionITSuite.scala | 221 +++++++++--------- .../src/main/antlr4/OpenSearchPPLParser.g4 | 2 + 6 files changed, 185 insertions(+), 179 deletions(-) create mode 100644 integ-test/script/data/relative_datetime.json create mode 100644 integ-test/script/data/relative_datetime.mapping.json diff --git a/docs/ppl-lang/functions/ppl-datetime.md b/docs/ppl-lang/functions/ppl-datetime.md index ba73c4d0d..7dbc7a471 100644 --- a/docs/ppl-lang/functions/ppl-datetime.md +++ b/docs/ppl-lang/functions/ppl-datetime.md @@ -411,30 +411,15 @@ Return type: BOOLEAN Example: - os> source=people | eval earliest = earliest("-1s", now()) | fields earliest | head 1 - fetched rows / total rows = 1/1 - +----------+ - | earliest | - |----------| - | True | - +----------+ - - os> source=people | eval earliest = earliest("now", now()) | fields earliest | head 1 - fetched rows / total rows = 1/1 - +----------+ - | earliest | - |----------| - | True | - +----------+ - - os> source=people | eval earliest = earliest("+1s", now()) | fields earliest | head 1 - fetched rows / total rows = 1/1 - +----------+ - | earliest | - |----------| - | False | - +----------+ - + os> source=relative_datetime | eval timestamp = relative_timestamp(relative_string) | where earliest("now",timestamp) | sort timestamp | fields description, relative_string + fetched rows / total rows = 3/3 + +--------------+-----------------+ + | description | relative_string | + +--------------+-----------------+ + | Now | NOW | + | Tomorrow | +D@D | + | In one month | +month | + +--------------+-----------------+ ### `FROM_UNIXTIME` @@ -560,29 +545,15 @@ Return type: BOOLEAN Example: - os> source=people | eval latest = latest("-1s", now()) | fields latest | head 1 - fetched rows / total rows = 1/1 - +--------+ - | latest | - |--------| - | False | - +--------+ - - os> source=people | eval latest = latest("now", now()) | fields latest | head 1 - fetched rows / total rows = 1/1 - +--------+ - | latest | - |--------| - | True | - +--------+ - - os> source=people | eval latest = latest("+1s", now()) | fields latest | head 1 - fetched rows / total rows = 1/1 - +--------+ - | latest | - |--------| - | True | - +--------+ + os> source=relative_datetime | eval timestamp = relative_timestamp(relative_string) | where latest("now",timestamp) | sort timestamp | fields description, relative_string + fetched rows / total rows = 3/3 + +---------------+-----------------+ + | description | relative_string | + +---------------+-----------------+ + | Two weeks ago | -2wk | + | Yesterday | -1d@d | + | Now | NOW | + +---------------+-----------------+ ### `LOCALTIMESTAMP` @@ -881,29 +852,17 @@ Return type: TIMESTAMP Example: - os> source=people | eval seconds_diff = timestampdiff(SECOND, now(), relative_timestamp("now")) | fields seconds_diff | head 1 - fetched rows / total rows = 1/1 - +--------------+ - | seconds_diff | - |--------------+ - | 0 | - +--------------+ - - os> source=people | eval hours_diff = timestampdiff(HOUR, now(), relative_timestamp("+1h")) | fields hours_diff | head 1 - fetched rows / total rows = 1/1 - +------------+ - | hours_diff | - |------------+ - | 1 | - +------------+ - - os> source=people | eval day = day_of_week(relative_timestamp("@w0")) | fields day | head 1 + os> source=relative_datetime | eval relative = relative_timestamp(relative_string) | sort relative | fields description, relative_string fetched rows / total rows = 1/1 - +-----+ - | day | - |-----| - | 1 | - +-----+ + +---------------+-----------------+ + | description | relative_string | + +---------------+-----------------+ + | Two weeks ago | -2wk | + | Yesterday | -1d@d | + | Now | NOW | + | Tomorrow | +D@D | + | In one month | +month | + +---------------+-----------------+ ### `SECOND` @@ -1227,7 +1186,7 @@ Examples:: **Description:** Usage: TIMESTAMPDIFF(interval, start, end) returns the difference between the start and end date/times in interval units. -Arguments will be automatically converted to a ]TIMESTAMP when appropriate. +Arguments will be automatically converted to a TIMESTAMP when appropriate. Any argument that is a STRING must be formatted as a valid TIMESTAMP. Argument type: INTERVAL, DATE/TIMESTAMP/STRING, DATE/TIMESTAMP/STRING diff --git a/integ-test/script/data/relative_datetime.json b/integ-test/script/data/relative_datetime.json new file mode 100644 index 000000000..ebd201c01 --- /dev/null +++ b/integ-test/script/data/relative_datetime.json @@ -0,0 +1,10 @@ +{"index": {"_index": "relative_datetime"}} +{"description": "Now", "relative_string": "NOW"} +{"index": {"_index": "relative_datetime"}} +{"description": "Tomorrow", "relative_string": "+D@D"} +{"index": {"_index": "relative_datetime"}} +{"description": "In one month", "relative_string": "+month"} +{"index": {"_index": "relative_datetime"}} +{"description": "Two weeks ago", "relative_string": "-2wk"} +{"index": {"_index": "relative_datetime"}} +{"description": "Yesterday", "relative_string": "-1d@d"} diff --git a/integ-test/script/data/relative_datetime.mapping.json b/integ-test/script/data/relative_datetime.mapping.json new file mode 100644 index 000000000..60c660b5a --- /dev/null +++ b/integ-test/script/data/relative_datetime.mapping.json @@ -0,0 +1,12 @@ +{ + "mappings": { + "properties": { + "description": { + "type": "text" + }, + "relative_string": { + "type": "text" + } + } + } +} \ No newline at end of file diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSuite.scala index 5ea123c9d..b09a82750 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSuite.scala @@ -229,6 +229,26 @@ trait FlintSparkSuite extends QueryTest with FlintSuite with OpenSearchSuite wit | """.stripMargin) } + protected def createRelativeDateTimeTable(testTable: String): Unit = { + sql(s""" + | CREATE TABLE $testTable + | ( + | description STRING, + | relative_string STRING + | ) + | USING $tableType $tableOptions + |""".stripMargin) + + sql(s""" + | INSERT INTO $testTable + | VALUES ('Now', 'NOW'), + | ('Tomorrow', '+D@D'), + | ('In one month', '+month'), + | ('Two weeks ago', '-2wk'), + | ('Yesterday', '-1d@d') + | """.stripMargin) + } + protected def createNullableStateCountryTable(testTable: String): Unit = { sql(s""" | CREATE TABLE $testTable diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala index bb8544ce6..b236bb9ad 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala @@ -22,13 +22,16 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite with StreamTest { /** Test table and index name */ - private val testTable = "spark_catalog.default.flint_ppl_test" + private val testPartitionedStateCountryTable = + "spark_catalog.default.flint_ppl_partitioned_state_country" + private val testRelativeDateTimeTable = "spark_catalog.default.flint_ppl_relative_datetime" override def beforeAll(): Unit = { super.beforeAll() - // Create test table - createPartitionedStateCountryTable(testTable) + // Create test tables. + createPartitionedStateCountryTable(testPartitionedStateCountryTable) + createRelativeDateTimeTable(testRelativeDateTimeTable) } protected override def afterEach(): Unit = { @@ -42,7 +45,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test adddate(date, numDays)") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `'2020-08-26' + 1` = ADDDATE(DATE('2020-08-26'), 1), `'2020-08-26' + (-1)` = ADDDATE(DATE('2020-08-26'), -1) | | fields `'2020-08-26' + 1`, `'2020-08-26' + (-1)` | head 1 | """.stripMargin) @@ -51,7 +54,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test subdate(date, numDays)") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `'2020-08-26' - 1` = SUBDATE(DATE('2020-08-26'), 1), `'2020-08-26' - (-1)` = SUBDATE(DATE('2020-08-26'), -1) | | fields `'2020-08-26' - 1`, `'2020-08-26' - (-1)` | head 1 | """.stripMargin) @@ -60,7 +63,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test CURRENT_DATE, CURDATE are synonyms") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `CURRENT_DATE` = CURRENT_DATE(), `CURDATE` = CURDATE() | | where CURRENT_DATE = CURDATE | | fields CURRENT_DATE, CURDATE | head 1 @@ -71,7 +74,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test LOCALTIME, LOCALTIMESTAMP, NOW are synonyms") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `LOCALTIME` = LOCALTIME(), `LOCALTIMESTAMP` = LOCALTIMESTAMP(), `NOW` = NOW() | | where LOCALTIME = LOCALTIMESTAMP and LOCALTIME = NOW | | fields LOCALTIME, LOCALTIMESTAMP, NOW | head 1 @@ -82,7 +85,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test DATE, TIMESTAMP") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `DATE('2020-08-26')` = DATE('2020-08-26') | | eval `DATE(TIMESTAMP('2020-08-26 13:49:00'))` = DATE(TIMESTAMP('2020-08-26 13:49:00')) | | eval `DATE('2020-08-26 13:49')` = DATE('2020-08-26 13:49') @@ -97,7 +100,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test DATE_FORMAT") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval format1 = DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), 'yyyy-MMM-dd hh:mm:ss a') | | eval format2 = DATE_FORMAT('1998-01-31 13:14:15.012345', 'HH:mm:ss.SSSSSS') | | fields format1, format2 @@ -108,7 +111,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test DATEDIFF") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval diff1 = DATEDIFF(DATE('2020-08-27'), DATE('2020-08-26')) | | eval diff2 = DATEDIFF(DATE('2020-08-26'), DATE('2020-08-27')) | | eval diff3 = DATEDIFF(DATE('2020-08-27'), DATE('2020-08-27')) @@ -123,7 +126,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test DAY, DAYOFMONTH, DAY_OF_MONTH are synonyms") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `DAY(DATE('2020-08-26'))` = DAY(DATE('2020-08-26')) | | eval `DAYOFMONTH(DATE('2020-08-26'))` = DAYOFMONTH(DATE('2020-08-26')) | | eval `DAY_OF_MONTH(DATE('2020-08-26'))` = DAY_OF_MONTH(DATE('2020-08-26')) @@ -135,7 +138,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test DAYOFWEEK, DAY_OF_WEEK are synonyms") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `DAYOFWEEK(DATE('2020-08-26'))` = DAYOFWEEK(DATE('2020-08-26')) | | eval `DAY_OF_WEEK(DATE('2020-08-26'))` = DAY_OF_WEEK(DATE('2020-08-26')) | | fields `DAYOFWEEK(DATE('2020-08-26'))`, `DAY_OF_WEEK(DATE('2020-08-26'))` @@ -146,7 +149,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test DAYOFYEAR, DAY_OF_YEAR are synonyms") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `DAY_OF_YEAR(DATE('2020-08-26'))` = DAY_OF_YEAR(DATE('2020-08-26')) | | eval `DAYOFYEAR(DATE('2020-08-26'))` = DAYOFYEAR(DATE('2020-08-26')) | | fields `DAY_OF_YEAR(DATE('2020-08-26'))`, `DAYOFYEAR(DATE('2020-08-26'))` @@ -157,7 +160,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test WEEK, WEEK_OF_YEAR are synonyms") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `WEEK(DATE('2008-02-20'))` = WEEK(DATE('2008-02-20')) | | eval `WEEK_OF_YEAR(DATE('2008-02-20'))` = WEEK_OF_YEAR(DATE('2008-02-20')) | | fields `WEEK(DATE('2008-02-20'))`, `WEEK_OF_YEAR(DATE('2008-02-20'))` @@ -168,7 +171,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test MONTH, MONTH_OF_YEAR are synonyms") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `MONTH(DATE('2020-08-26'))` = MONTH(DATE('2020-08-26')) | | eval `MONTH_OF_YEAR(DATE('2020-08-26'))` = MONTH_OF_YEAR(DATE('2020-08-26')) | | fields `MONTH(DATE('2020-08-26'))`, `MONTH_OF_YEAR(DATE('2020-08-26'))` @@ -178,7 +181,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite } test("test WEEKDAY") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `weekday(DATE('2020-08-26'))` = weekday(DATE('2020-08-26')) | | eval `weekday(DATE('2020-08-27'))` = weekday(DATE('2020-08-27')) | | fields `weekday(DATE('2020-08-26'))`, `weekday(DATE('2020-08-27'))` @@ -189,7 +192,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test YEAR") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `YEAR(DATE('2020-08-26'))` = YEAR(DATE('2020-08-26')) | fields `YEAR(DATE('2020-08-26'))` | | head 1 | """.stripMargin) @@ -198,14 +201,15 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test from_unixtime and unix_timestamp") { val frame = sql(s""" - | source = $testTable |where unix_timestamp(from_unixtime(1700000001)) > 1700000000 | fields name, age + | source = $testPartitionedStateCountryTable |where unix_timestamp(from_unixtime(1700000001)) > 1700000000 | fields name, age | """.stripMargin) assertSameRows( Seq(Row("Jake", 70), Row("Hello", 30), Row("John", 25), Row("Jane", 20)), frame) val logicalPlan: LogicalPlan = frame.queryExecution.logical - val table = UnresolvedRelation(Seq("spark_catalog", "default", "flint_ppl_test")) + val table = + UnresolvedRelation(Seq("spark_catalog", "default", "flint_ppl_partitioned_state_country")) val filterExpr = GreaterThan( UnresolvedFunction( "unix_timestamp", @@ -220,31 +224,31 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test DATE_ADD") { val frame1 = sql(s""" - | source = $testTable | eval `'2020-08-26' + 2d` = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 DAY) + | source = $testPartitionedStateCountryTable | eval `'2020-08-26' + 2d` = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 DAY) | | fields `'2020-08-26' + 2d` | head 1 | """.stripMargin) assertSameRows(Seq(Row(Date.valueOf("2020-08-28"))), frame1) val frame2 = sql(s""" - | source = $testTable | eval `'2020-08-26' - 2d` = DATE_ADD(DATE('2020-08-26'), INTERVAL -2 DAY) + | source = $testPartitionedStateCountryTable | eval `'2020-08-26' - 2d` = DATE_ADD(DATE('2020-08-26'), INTERVAL -2 DAY) | | fields `'2020-08-26' - 2d` | head 1 | """.stripMargin) assertSameRows(Seq(Row(Date.valueOf("2020-08-24"))), frame2) val frame3 = sql(s""" - | source = $testTable | eval `'2020-08-26' + 2m` = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 MONTH) + | source = $testPartitionedStateCountryTable | eval `'2020-08-26' + 2m` = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 MONTH) | | fields `'2020-08-26' + 2m` | head 1 | """.stripMargin) assertSameRows(Seq(Row(Date.valueOf("2020-10-26"))), frame3) val frame4 = sql(s""" - | source = $testTable | eval `'2020-08-26' + 2y` = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 YEAR) + | source = $testPartitionedStateCountryTable | eval `'2020-08-26' + 2y` = DATE_ADD(DATE('2020-08-26'), INTERVAL 2 YEAR) | | fields `'2020-08-26' + 2y` | head 1 | """.stripMargin) assertSameRows(Seq(Row(Date.valueOf("2022-08-26"))), frame4) val ex = intercept[AnalysisException](sql(s""" - | source = $testTable | eval `'2020-08-26 01:01:01' + 2h` = DATE_ADD(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 2 HOUR) + | source = $testPartitionedStateCountryTable | eval `'2020-08-26 01:01:01' + 2h` = DATE_ADD(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 2 HOUR) | | fields `'2020-08-26 01:01:01' + 2h` | head 1 | """.stripMargin)) assert(ex.getMessage.contains("""Parameter 1 requires the "DATE" type""")) @@ -252,31 +256,31 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test DATE_SUB") { val frame1 = sql(s""" - | source = $testTable | eval `'2020-08-26' - 2d` = DATE_SUB(DATE('2020-08-26'), INTERVAL 2 DAY) + | source = $testPartitionedStateCountryTable | eval `'2020-08-26' - 2d` = DATE_SUB(DATE('2020-08-26'), INTERVAL 2 DAY) | | fields `'2020-08-26' - 2d` | head 1 | """.stripMargin) assertSameRows(Seq(Row(Date.valueOf("2020-08-24"))), frame1) val frame2 = sql(s""" - | source = $testTable | eval `'2020-08-26' + 2d` = DATE_SUB(DATE('2020-08-26'), INTERVAL -2 DAY) + | source = $testPartitionedStateCountryTable | eval `'2020-08-26' + 2d` = DATE_SUB(DATE('2020-08-26'), INTERVAL -2 DAY) | | fields `'2020-08-26' + 2d` | head 1 | """.stripMargin) assertSameRows(Seq(Row(Date.valueOf("2020-08-28"))), frame2) val frame3 = sql(s""" - | source = $testTable | eval `'2020-08-26' - 2m` = DATE_SUB(DATE('2020-08-26'), INTERVAL 12 MONTH) + | source = $testPartitionedStateCountryTable | eval `'2020-08-26' - 2m` = DATE_SUB(DATE('2020-08-26'), INTERVAL 12 MONTH) | | fields `'2020-08-26' - 2m` | head 1 | """.stripMargin) assertSameRows(Seq(Row(Date.valueOf("2019-08-26"))), frame3) val frame4 = sql(s""" - | source = $testTable | eval `'2020-08-26' - 2y` = DATE_SUB(DATE('2020-08-26'), INTERVAL 2 YEAR) + | source = $testPartitionedStateCountryTable | eval `'2020-08-26' - 2y` = DATE_SUB(DATE('2020-08-26'), INTERVAL 2 YEAR) | | fields `'2020-08-26' - 2y` | head 1 | """.stripMargin) assertSameRows(Seq(Row(Date.valueOf("2018-08-26"))), frame4) val ex = intercept[AnalysisException](sql(s""" - | source = $testTable | eval `'2020-08-26 01:01:01' - 2h` = DATE_SUB(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 2 HOUR) + | source = $testPartitionedStateCountryTable | eval `'2020-08-26 01:01:01' - 2h` = DATE_SUB(TIMESTAMP('2020-08-26 01:01:01'), INTERVAL 2 HOUR) | | fields `'2020-08-26 01:01:01' - 2h` | head 1 | """.stripMargin)) assert(ex.getMessage.contains("""Parameter 1 requires the "DATE" type""")) @@ -284,7 +288,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test TIMESTAMPADD") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')` = TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | | eval `TIMESTAMPADD(DAY, 17, TIMESTAMP('2000-01-01 00:00:00'))` = TIMESTAMPADD(DAY, 17, TIMESTAMP('2000-01-01 00:00:00')) | | eval `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` = TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') @@ -302,7 +306,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test TIMESTAMPDIFF") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')` = TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | | eval `TIMESTAMPDIFF(SECOND, TIMESTAMP('2000-01-01 00:00:23'), TIMESTAMP('2000-01-01 00:00:00'))` = TIMESTAMPDIFF(SECOND, TIMESTAMP('2000-01-01 00:00:23'), TIMESTAMP('2000-01-01 00:00:00')) | | fields `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')`, `TIMESTAMPDIFF(SECOND, TIMESTAMP('2000-01-01 00:00:23'), TIMESTAMP('2000-01-01 00:00:00'))` @@ -313,7 +317,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test CURRENT_TIMEZONE") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `CURRENT_TIMEZONE` = CURRENT_TIMEZONE() | | fields `CURRENT_TIMEZONE` | """.stripMargin) @@ -322,7 +326,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test UTC_TIMESTAMP") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `UTC_TIMESTAMP` = UTC_TIMESTAMP() | | fields `UTC_TIMESTAMP` | """.stripMargin) @@ -331,7 +335,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test hour, minute, second, HOUR_OF_DAY, MINUTE_OF_HOUR") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval h = hour(timestamp('01:02:03')), m = minute(timestamp('01:02:03')), s = second(timestamp('01:02:03')) | | eval hs = hour('2024-07-30 01:02:03'), ms = minute('2024-07-30 01:02:03'), ss = second('01:02:03') | | eval h_d = HOUR_OF_DAY(timestamp('01:02:03')), m_h = MINUTE_OF_HOUR(timestamp('01:02:03')), s_m = SECOND_OF_MINUTE(timestamp('01:02:03')) @@ -342,7 +346,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test LAST_DAY") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `last_day('2023-02-06')` = last_day('2023-02-06') | | fields `last_day('2023-02-06')` | | head 1 @@ -352,7 +356,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test MAKE_DATE") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `MAKE_DATE(1945, 5, 9)` = MAKE_DATE(1945, 5, 9) | fields `MAKE_DATE(1945, 5, 9)` | | head 1 | """.stripMargin) @@ -361,7 +365,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test QUARTER") { val frame = sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `QUARTER(DATE('2020-08-26'))` = QUARTER(DATE('2020-08-26')) | fields `QUARTER(DATE('2020-08-26'))` | | head 1 | """.stripMargin) @@ -369,70 +373,69 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite } test("test RELATIVE_TIMESTAMP") { - var frame = sql(s""" - | source=$testTable - | | eval seconds_diff = timestampdiff(SECOND, now(), relative_timestamp("now")) - | | fields seconds_diff - | | head 1 - | """.stripMargin) - assertSameRows(Seq(Row(0)), frame) - - frame = sql(s""" - | source=$testTable - | | eval hours_diff = timestampdiff(HOUR, relative_timestamp("+1h"), relative_timestamp("+1d")) - | | fields hours_diff - | | head 1 + val frame = sql(s""" + | source=$testRelativeDateTimeTable + | | eval relative = relative_timestamp(relative_string) + | | sort relative + | | fields description, relative_string | """.stripMargin) - assertSameRows(Seq(Row(23)), frame) - - frame = sql(s""" - | source=$testTable - | | eval day = day_of_week(relative_timestamp("@w0")) - | | fields day - | | head 1 - | """.stripMargin) - assertSameRows(Seq(Row(1)), frame) - - frame = sql(s""" - | source=$testTable - | | eval last_wednesday = relative_timestamp("-1d@w3") - | | eval actual_days_ago = timestampdiff(DAY, last_wednesday, now()) - | | eval day_of_week = day_of_week(now()) - | | eval expected_days_ago = case(day_of_week > 4, day_of_week - 4 else day_of_week + 3) - | | eval test_result = (expected_days_ago = actual_days_ago) - | | fields test_result - | | head 1 - | """.stripMargin) - assertSameRows(Seq(Row(true)), frame) + assertSameRows( + Seq( + Row("Two weeks ago", "-2wk"), + Row("Yesterday", "-1d@d"), + Row("Now", "NOW"), + Row("Tomorrow", "+D@D"), + Row("In one month", "+month")), + frame) } test("test EARLIEST") { var frame = sql(s""" - | source=$testTable - | | eval earliest_second_before = earliest("-1s", now()) - | | eval earliest_now = earliest("now", now()) - | | eval earliest_second_after = earliest("+1s", now()) - | | fields earliest_second_before, earliest_now, earliest_second_after - | | head 1 + | source=$testRelativeDateTimeTable + | | eval relative = relative_timestamp(relative_string) + | | sort relative + | | where earliest("now",relative) + | | fields description, relative_string + | """.stripMargin) + assertSameRows( + Seq(Row("Now", "NOW"), Row("Tomorrow", "+D@D"), Row("In one month", "+month")), + frame) + + frame = sql(s""" + | source=$testRelativeDateTimeTable + | | eval relative = relative_timestamp(relative_string) + | | sort relative + | | where earliest("+2days",relative) + | | fields description, relative_string | """.stripMargin) - assertSameRows(Seq(Row(true, true, false)), frame) + assertSameRows(Seq(Row("In one month", "+month")), frame) } test("test LATEST") { var frame = sql(s""" - | source=$testTable - | | eval latest_second_before = latest("-1s", now()) - | | eval latest_now = latest("now", now()) - | | eval latest_second_after = latest("+1s", now()) - | | fields latest_second_before, latest_now, latest_second_after - | | head 1 + | source=$testRelativeDateTimeTable + | | eval relative = relative_timestamp(relative_string) + | | sort relative + | | where latest("now",relative) + | | fields description, relative_string | """.stripMargin) - assertSameRows(Seq(Row(false, true, true)), frame) + assertSameRows( + Seq(Row("Two weeks ago", "-2wk"), Row("Yesterday", "-1d@d"), Row("Now", "NOW")), + frame) + + frame = sql(s""" + | source=$testRelativeDateTimeTable + | | eval relative = relative_timestamp(relative_string) + | | sort relative + | | where latest("-2days",relative) + | | fields description, relative_string + | """.stripMargin) + assertSameRows(Seq(Row("Two weeks ago", "-2wk")), frame) } test("test CURRENT_TIME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `CURRENT_TIME` = CURRENT_TIME() | | fields CURRENT_TIME | head 1 | """.stripMargin)) @@ -441,7 +444,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test CONVERT_TZ is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `CONVERT_TZ` = CONVERT_TZ() | | fields CONVERT_TZ | head 1 | """.stripMargin)) @@ -450,7 +453,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test ADDTIME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `ADDTIME` = ADDTIME() | | fields ADDTIME | head 1 | """.stripMargin)) @@ -459,7 +462,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test DATETIME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `DATETIME` = DATETIME() | | fields DATETIME | head 1 | """.stripMargin)) @@ -468,7 +471,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test DAYNAME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `DAYNAME` = DAYNAME() | | fields DAYNAME | head 1 | """.stripMargin)) @@ -477,7 +480,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test FROM_DAYS is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `FROM_DAYS` = FROM_DAYS() | | fields FROM_DAYS | head 1 | """.stripMargin)) @@ -486,7 +489,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test GET_FORMAT is not supported") { intercept[Exception](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `GET_FORMAT` = GET_FORMAT(DATE, 'USA') | | fields GET_FORMAT | head 1 | """.stripMargin)) @@ -494,7 +497,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test MAKETIME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `MAKETIME` = MAKETIME() | | fields MAKETIME | head 1 | """.stripMargin)) @@ -503,7 +506,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test MICROSECOND is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `MICROSECOND` = MICROSECOND() | | fields MICROSECOND | head 1 | """.stripMargin)) @@ -512,7 +515,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test MINUTE_OF_DAY is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `MINUTE_OF_DAY` = MINUTE_OF_DAY() | | fields MINUTE_OF_DAY | head 1 | """.stripMargin)) @@ -521,7 +524,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test PERIOD_ADD is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `PERIOD_ADD` = PERIOD_ADD() | | fields PERIOD_ADD | head 1 | """.stripMargin)) @@ -530,7 +533,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test PERIOD_DIFF is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `PERIOD_DIFF` = PERIOD_DIFF() | | fields PERIOD_DIFF | head 1 | """.stripMargin)) @@ -539,7 +542,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test SEC_TO_TIME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `SEC_TO_TIME` = SEC_TO_TIME() | | fields SEC_TO_TIME | head 1 | """.stripMargin)) @@ -548,7 +551,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test STR_TO_DATE is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `STR_TO_DATE` = STR_TO_DATE() | | fields STR_TO_DATE | head 1 | """.stripMargin)) @@ -557,7 +560,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test SUBTIME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `SUBTIME` = SUBTIME() | | fields SUBTIME | head 1 | """.stripMargin)) @@ -566,7 +569,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test TIME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `TIME` = TIME() | | fields TIME | head 1 | """.stripMargin)) @@ -575,7 +578,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test TIME_FORMAT is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `TIME_FORMAT` = TIME_FORMAT() | | fields TIME_FORMAT | head 1 | """.stripMargin)) @@ -584,7 +587,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test TIME_TO_SEC is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `TIME_TO_SEC` = TIME_TO_SEC() | | fields TIME_TO_SEC | head 1 | """.stripMargin)) @@ -593,7 +596,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test TIMEDIFF is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `TIMEDIFF` = TIMEDIFF() | | fields TIMEDIFF | head 1 | """.stripMargin)) @@ -602,7 +605,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test TO_DAYS is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `TO_DAYS` = TO_DAYS() | | fields TO_DAYS | head 1 | """.stripMargin)) @@ -611,7 +614,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test TO_SECONDS is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `TO_SECONDS` = TO_SECONDS() | | fields TO_SECONDS | head 1 | """.stripMargin)) @@ -620,7 +623,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test UTC_DATE is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `UTC_DATE` = UTC_DATE() | | fields UTC_DATE | head 1 | """.stripMargin)) @@ -629,7 +632,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test UTC_TIME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `UTC_TIME` = UTC_TIME() | | fields UTC_TIME | head 1 | """.stripMargin)) @@ -638,7 +641,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite test("test YEARWEEK is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" - | source = $testTable + | source = $testPartitionedStateCountryTable | | eval `YEARWEEK` = YEARWEEK() | | fields YEARWEEK | head 1 | """.stripMargin)) diff --git a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 index dc5d44c57..4ea9e21f1 100644 --- a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 +++ b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 @@ -853,6 +853,8 @@ conditionFunctionBase | NULLIF | ISPRESENT | JSON_VALID + | EARLIEST + | LATEST ; systemFunctionName