From 434f40355d59ac189baec17337b1b39a4e8852ee Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sat, 29 Apr 2017 22:38:26 +0900 Subject: [PATCH 01/16] The MySQL plugin compare server and client timezone --- .../input/jdbc/AbstractJdbcInputPlugin.java | 2 ++ .../input/jdbc/JdbcInputConnection.java | 7 +++++ .../input/mysql/MySQLInputConnection.java | 31 +++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java b/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java index b0605f51..29ec9431 100644 --- a/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java +++ b/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java @@ -426,6 +426,8 @@ public TaskReport run(TaskSource taskSource, LastRecordStore lastRecordStore = null; try (JdbcInputConnection con = newConnection(task)) { + con.before_load(); + List getters = newColumnGetters(con, task, querySchema, pageBuilder); try (BatchSelect cursor = con.newSelectCursor(builtQuery, getters, task.getFetchRows(), task.getSocketTimeout())) { while (true) { diff --git a/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java b/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java index a27adecc..f0569c9a 100644 --- a/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java +++ b/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java @@ -386,4 +386,11 @@ private Set getColumnNames(String tableName) throws SQLException return columnNamesBuilder.build(); } } + + public void before_load() + throws SQLException + { + + } + } diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java index b783db5e..31a89b16 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java @@ -1,10 +1,12 @@ package org.embulk.input.mysql; +import java.sql.Statement; import java.util.List; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.ResultSet; +import java.util.Locale; import java.util.TimeZone; import com.mysql.jdbc.ConnectionImpl; @@ -59,4 +61,33 @@ public TimeZone getServerTimezoneTZ() { return ((ConnectionImpl) connection).getServerTimezoneTZ(); } + + @Override + public void before_load() + throws SQLException + { + Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery("select @@system_time_zone"); + + if( rs.next() ) { + String sys_tz_name = rs.getString(1); + TimeZone sys_tz = TimeZone.getTimeZone(sys_tz_name); + + String usr_tz_name = System.getProperty("user.timezone"); + TimeZone usr_tz = TimeZone.getTimeZone(usr_tz_name); + + if( !sys_tz.hasSameRules(usr_tz) ) { + logger.warn(String.format(Locale.ENGLISH, + "The server timezone and client timezone are different, the plugin will fetch wrong datetime values.")); + logger.warn(String.format(Locale.ENGLISH, + "Use `options: { useLegacyDatetimeCode: false }`")); + } + logger.warn(String.format(Locale.ENGLISH,"The plugin will set `useLegacyDatetimeCode=false` by default in future.")); + + } + else { + // TODO Error check + } + + } } From 9a3fd6553924a25c9edd16f7e293a1d76c9dc383 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 3 May 2017 23:13:55 +0900 Subject: [PATCH 02/16] Calculate GMT offset instead of @@system_time_zone --- .../input/mysql/MySQLInputConnection.java | 55 +++++++++++++------ 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java index 31a89b16..5e5c9104 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java @@ -66,28 +66,49 @@ public TimeZone getServerTimezoneTZ() public void before_load() throws SQLException { - Statement stmt = connection.createStatement(); - ResultSet rs = stmt.executeQuery("select @@system_time_zone"); + String svr_tz_name = getServerTimeZoneOffset(); + TimeZone svr_tz = TimeZone.getTimeZone(svr_tz_name); - if( rs.next() ) { - String sys_tz_name = rs.getString(1); - TimeZone sys_tz = TimeZone.getTimeZone(sys_tz_name); + String usr_tz_name = System.getProperty("user.timezone"); + TimeZone usr_tz = TimeZone.getTimeZone(usr_tz_name); - String usr_tz_name = System.getProperty("user.timezone"); - TimeZone usr_tz = TimeZone.getTimeZone(usr_tz_name); + if( svr_tz.getRawOffset() != usr_tz.getRawOffset() ) { + logger.warn(String.format(Locale.ENGLISH, + "The server timezone offset(%s) and client timezone has different timezone offset. The plugin will fetch wrong datetime values.",svr_tz_name,usr_tz_name)); + logger.warn(String.format(Locale.ENGLISH, + "Use `options: { useLegacyDatetimeCode: false }`")); + } + logger.warn(String.format(Locale.ENGLISH,"The plugin will set `useLegacyDatetimeCode=false` by default in future.")); + } - if( !sys_tz.hasSameRules(usr_tz) ) { - logger.warn(String.format(Locale.ENGLISH, - "The server timezone and client timezone are different, the plugin will fetch wrong datetime values.")); - logger.warn(String.format(Locale.ENGLISH, - "Use `options: { useLegacyDatetimeCode: false }`")); - } - logger.warn(String.format(Locale.ENGLISH,"The plugin will set `useLegacyDatetimeCode=false` by default in future.")); + private String getServerTimeZoneOffset(){ + + // + // First, I used `@@system_time_zone`. but It return non Time Zone Abbreviations name on a specific platform. + // So, This method get GMT offset with query. + // + String query = "select IF(timediff(now(),utc_timestamp())=0,'UTC',\n" + + " IF(timediff(now(),utc_timestamp())>0,\n" + + " CONCAT('GMT+',TIME_FORMAT(timediff(now(),utc_timestamp()),'%h:%m')),\n" + + " CONCAT('GMT',TIME_FORMAT(timediff( now(), utc_timestamp()),'%h:%m')))) as offset;\n"; + try { + Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(query); + + if (rs.next()) { + String tz_name = rs.getString(1); + logger.info(String.format(Locale.ENGLISH,"Server timezone offset: %s",tz_name)); + return tz_name; + } + else { + // TODO Error Test. + return null; + } } - else { - // TODO Error check + catch (SQLException ex) { + // TODO + return null; } - } } From a30a169ae8e9c2220e885cf9b6cf0736895ee9d9 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 3 May 2017 23:16:39 +0900 Subject: [PATCH 03/16] Output log client timezone name --- .../main/java/org/embulk/input/mysql/MySQLInputConnection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java index 5e5c9104..3f1dce95 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java @@ -74,7 +74,7 @@ public void before_load() if( svr_tz.getRawOffset() != usr_tz.getRawOffset() ) { logger.warn(String.format(Locale.ENGLISH, - "The server timezone offset(%s) and client timezone has different timezone offset. The plugin will fetch wrong datetime values.",svr_tz_name,usr_tz_name)); + "The server timezone offset(%s) and client timezone(%s) has different timezone offset. The plugin will fetch wrong datetime values.",svr_tz_name,usr_tz_name)); logger.warn(String.format(Locale.ENGLISH, "Use `options: { useLegacyDatetimeCode: false }`")); } From 98a2bec0f0521f197fd8d5a6de831dd607390aa9 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 10 May 2017 13:13:28 +0900 Subject: [PATCH 04/16] Use offset second instead of offset name --- .../input/mysql/MySQLInputConnection.java | 67 ++++++++++++------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java index 3f1dce95..641199bc 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java @@ -66,49 +66,70 @@ public TimeZone getServerTimezoneTZ() public void before_load() throws SQLException { - String svr_tz_name = getServerTimeZoneOffset(); - TimeZone svr_tz = TimeZone.getTimeZone(svr_tz_name); - - String usr_tz_name = System.getProperty("user.timezone"); - TimeZone usr_tz = TimeZone.getTimeZone(usr_tz_name); - - if( svr_tz.getRawOffset() != usr_tz.getRawOffset() ) { - logger.warn(String.format(Locale.ENGLISH, - "The server timezone offset(%s) and client timezone(%s) has different timezone offset. The plugin will fetch wrong datetime values.",svr_tz_name,usr_tz_name)); - logger.warn(String.format(Locale.ENGLISH, - "Use `options: { useLegacyDatetimeCode: false }`")); - } - logger.warn(String.format(Locale.ENGLISH,"The plugin will set `useLegacyDatetimeCode=false` by default in future.")); + compareTimeZoneOffset(); } - private String getServerTimeZoneOffset(){ + private int getServerTimeZoneOffsetInSeconds(){ // // First, I used `@@system_time_zone`. but It return non Time Zone Abbreviations name on a specific platform. // So, This method get GMT offset with query. // - String query = "select IF(timediff(now(),utc_timestamp())=0,'UTC',\n" + - " IF(timediff(now(),utc_timestamp())>0,\n" + - " CONCAT('GMT+',TIME_FORMAT(timediff(now(),utc_timestamp()),'%h:%m')),\n" + - " CONCAT('GMT',TIME_FORMAT(timediff( now(), utc_timestamp()),'%h:%m')))) as offset;\n"; + String query = "select TIME_TO_SEC(timediff(now(),utc_timestamp()));"; try { Statement stmt = connection.createStatement(); ResultSet rs = stmt.executeQuery(query); if (rs.next()) { - String tz_name = rs.getString(1); - logger.info(String.format(Locale.ENGLISH,"Server timezone offset: %s",tz_name)); - return tz_name; + int offset = rs.getInt(1); + logger.info(String.format(Locale.ENGLISH,"Server timezone offset in seconds: %d",offset)); + return offset; } else { // TODO Error Test. - return null; + return -1; } } catch (SQLException ex) { // TODO - return null; + return -1; + } + } + + private void compareTimeZoneOffset(){ + + int tz_offset_sec = getServerTimeZoneOffsetInSeconds(); + String svr_tz_name = makeTimeZoneOffsetFromSeconds(tz_offset_sec); + TimeZone svr_tz = TimeZone.getTimeZone(svr_tz_name); + + String usr_tz_name = System.getProperty("user.timezone"); + TimeZone usr_tz = TimeZone.getTimeZone(usr_tz_name); + + if( svr_tz.getRawOffset() != usr_tz.getRawOffset() ) { + logger.warn(String.format(Locale.ENGLISH, + "The server timezone offset(%s) and client timezone(%s) has different timezone offset. The plugin will fetch wrong datetime values.",svr_tz_name,usr_tz_name)); + logger.warn(String.format(Locale.ENGLISH, + "Use `options: { useLegacyDatetimeCode: false }`")); + } + logger.warn(String.format(Locale.ENGLISH,"The plugin will set `useLegacyDatetimeCode=false` by default in future.")); + + } + + private String makeTimeZoneOffsetFromSeconds(int tz_offset_sec) + { + if( tz_offset_sec == 0 ){ + return "UTC"; + } + else { + int hour_sec = 3600; + int min_sec = 60; + + String sign = tz_offset_sec > 0 ? "+" : "-"; + int abs_offset_sec = Math.abs(tz_offset_sec); + int tz_hour = abs_offset_sec / hour_sec; + int tz_min = abs_offset_sec % hour_sec / min_sec; + return String.format(Locale.ENGLISH,"GMT%s%02d:%02d",sign,tz_hour,tz_min); } } } From a8ad95b6a933f1919e2f5ddd2813b76e10229e3a Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 10 May 2017 17:38:06 +0900 Subject: [PATCH 05/16] Add MySQLTimeZoneBuilder class for build TimeZone --- .../embulk/input/MySQLTimeZoneBuilder.java | 46 +++++++++++++ .../input/mysql/MySQLInputConnection.java | 65 ++++--------------- 2 files changed, 57 insertions(+), 54 deletions(-) create mode 100644 embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java new file mode 100644 index 00000000..1989988f --- /dev/null +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java @@ -0,0 +1,46 @@ +package org.embulk.input; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Locale; +import java.util.TimeZone; + +public class MySQLTimeZoneBuilder +{ + public static final int ONE_HOUR_SEC = 3600; + public static final int ONE_MIN_SEC = 60; + + public static TimeZone fromSystemTimeZone(Connection connection) + throws SQLException + { + // + // First, I used `@@system_time_zone`. but It return non Time Zone Abbreviations name on a specific platform. + // So, This method calculate GMT offset with query. + // + String query = "select TIME_TO_SEC(timediff(now(),utc_timestamp()));"; + Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(query); + + if (rs.next()) { + int offset_seconds = rs.getInt(1); + return fromGMTOffsetSeconds(offset_seconds); + } + else { + // TODO Error check. + return null; + } + } + + public static TimeZone fromGMTOffsetSeconds(int offset_seconds) + { + String sign = offset_seconds > 0 ? "+" : "-"; + int abs_offset_sec = Math.abs(offset_seconds); + int tz_hour = abs_offset_sec / ONE_HOUR_SEC; + int tz_min = abs_offset_sec % ONE_HOUR_SEC / ONE_MIN_SEC; + String tz_name = String.format(Locale.ENGLISH,"GMT%s%02d:%02d",sign,tz_hour,tz_min); + + return TimeZone.getTimeZone(tz_name); + } +} diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java index 641199bc..656afbb7 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java @@ -11,6 +11,7 @@ import com.mysql.jdbc.ConnectionImpl; import com.mysql.jdbc.ConnectionProperties; +import org.embulk.input.MySQLTimeZoneBuilder; import org.embulk.input.jdbc.JdbcInputConnection; import org.embulk.input.jdbc.JdbcLiteral; import org.embulk.input.jdbc.getter.ColumnGetter; @@ -66,70 +67,26 @@ public TimeZone getServerTimezoneTZ() public void before_load() throws SQLException { - compareTimeZoneOffset(); - } - - private int getServerTimeZoneOffsetInSeconds(){ - - // - // First, I used `@@system_time_zone`. but It return non Time Zone Abbreviations name on a specific platform. - // So, This method get GMT offset with query. - // - String query = "select TIME_TO_SEC(timediff(now(),utc_timestamp()));"; - - try { - Statement stmt = connection.createStatement(); - ResultSet rs = stmt.executeQuery(query); - - if (rs.next()) { - int offset = rs.getInt(1); - logger.info(String.format(Locale.ENGLISH,"Server timezone offset in seconds: %d",offset)); - return offset; - } - else { - // TODO Error Test. - return -1; - } - } - catch (SQLException ex) { - // TODO - return -1; - } - } - - private void compareTimeZoneOffset(){ - - int tz_offset_sec = getServerTimeZoneOffsetInSeconds(); - String svr_tz_name = makeTimeZoneOffsetFromSeconds(tz_offset_sec); - TimeZone svr_tz = TimeZone.getTimeZone(svr_tz_name); + // TODO error check. + TimeZone svr_tz = MySQLTimeZoneBuilder.fromSystemTimeZone(connection); String usr_tz_name = System.getProperty("user.timezone"); TimeZone usr_tz = TimeZone.getTimeZone(usr_tz_name); + // + // Compare offset only. Although I expect to return true, the following code return false, + // + // TimeZone tz_jst = TimeZone.getTimeZone("JST"); + // TimeZone tz_gmt9 = TimeZone.getTimeZone("GMT+9"); + // tz_jst.hasSameRules(tz_gmt9) // return false. + // if( svr_tz.getRawOffset() != usr_tz.getRawOffset() ) { logger.warn(String.format(Locale.ENGLISH, - "The server timezone offset(%s) and client timezone(%s) has different timezone offset. The plugin will fetch wrong datetime values.",svr_tz_name,usr_tz_name)); + "The server timezone offset(%s) and client timezone(%s) has different timezone offset. The plugin will fetch wrong datetime values.",svr_tz.getDisplayName(),usr_tz_name)); logger.warn(String.format(Locale.ENGLISH, "Use `options: { useLegacyDatetimeCode: false }`")); } logger.warn(String.format(Locale.ENGLISH,"The plugin will set `useLegacyDatetimeCode=false` by default in future.")); - } - private String makeTimeZoneOffsetFromSeconds(int tz_offset_sec) - { - if( tz_offset_sec == 0 ){ - return "UTC"; - } - else { - int hour_sec = 3600; - int min_sec = 60; - - String sign = tz_offset_sec > 0 ? "+" : "-"; - int abs_offset_sec = Math.abs(tz_offset_sec); - int tz_hour = abs_offset_sec / hour_sec; - int tz_min = abs_offset_sec % hour_sec / min_sec; - return String.format(Locale.ENGLISH,"GMT%s%02d:%02d",sign,tz_hour,tz_min); - } - } } From 091d65c48172436f998aedb644d5563b7fe9fbf8 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 10 May 2017 17:44:56 +0900 Subject: [PATCH 06/16] Change parameter scope to private --- .../src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java index 1989988f..c1103a01 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java @@ -9,8 +9,8 @@ public class MySQLTimeZoneBuilder { - public static final int ONE_HOUR_SEC = 3600; - public static final int ONE_MIN_SEC = 60; + private static final int ONE_HOUR_SEC = 3600; + private static final int ONE_MIN_SEC = 60; public static TimeZone fromSystemTimeZone(Connection connection) throws SQLException From 14b8b5ea907f2e8aa06eda321c7146444193c61c Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 10 May 2017 17:48:10 +0900 Subject: [PATCH 07/16] Return UTC timezone if offset is zero --- .../java/org/embulk/input/MySQLTimeZoneBuilder.java | 13 ++++++++----- .../embulk/input/mysql/MySQLInputConnection.java | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java index c1103a01..f49953f3 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java @@ -35,12 +35,15 @@ public static TimeZone fromSystemTimeZone(Connection connection) public static TimeZone fromGMTOffsetSeconds(int offset_seconds) { - String sign = offset_seconds > 0 ? "+" : "-"; - int abs_offset_sec = Math.abs(offset_seconds); - int tz_hour = abs_offset_sec / ONE_HOUR_SEC; - int tz_min = abs_offset_sec % ONE_HOUR_SEC / ONE_MIN_SEC; - String tz_name = String.format(Locale.ENGLISH,"GMT%s%02d:%02d",sign,tz_hour,tz_min); + if( offset_seconds == 0 ) { + return TimeZone.getTimeZone("UTC"); + } + String sign = offset_seconds > 0 ? "+" : "-"; + int abs_offset_sec = Math.abs(offset_seconds); + int tz_hour = abs_offset_sec / ONE_HOUR_SEC; + int tz_min = abs_offset_sec % ONE_HOUR_SEC / ONE_MIN_SEC; + String tz_name = String.format(Locale.ENGLISH, "GMT%s%02d:%02d", sign, tz_hour, tz_min); return TimeZone.getTimeZone(tz_name); } } diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java index 656afbb7..e5788874 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java @@ -82,7 +82,7 @@ public void before_load() // if( svr_tz.getRawOffset() != usr_tz.getRawOffset() ) { logger.warn(String.format(Locale.ENGLISH, - "The server timezone offset(%s) and client timezone(%s) has different timezone offset. The plugin will fetch wrong datetime values.",svr_tz.getDisplayName(),usr_tz_name)); + "The server timezone offset(%s) and client timezone(%s) has different timezone offset. The plugin will fetch wrong datetime values.",svr_tz.getID(),usr_tz_name)); logger.warn(String.format(Locale.ENGLISH, "Use `options: { useLegacyDatetimeCode: false }`")); } From 6ba1f85861327ae3d3e9c666a78ac6ee67667a74 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Fri, 12 May 2017 21:12:47 +0900 Subject: [PATCH 08/16] Reimplement compareTimeZone --- .../input/jdbc/AbstractJdbcInputPlugin.java | 4 +- .../input/jdbc/JdbcInputConnection.java | 7 ---- .../org/embulk/input/MySQLInputPlugin.java | 11 +++++ .../embulk/input/MySQLTimeZoneBuilder.java | 40 ++++++++++--------- .../input/mysql/MySQLInputConnection.java | 20 ++++++---- 5 files changed, 46 insertions(+), 36 deletions(-) diff --git a/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java b/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java index 29ec9431..6b3ee9ac 100644 --- a/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java +++ b/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java @@ -193,7 +193,7 @@ public ConfigDiff transaction(ConfigSource config, return buildNextConfigDiff(task, control.run(task.dump(), schema, 1)); } - private Schema setupTask(JdbcInputConnection con, PluginTask task) throws SQLException + protected Schema setupTask(JdbcInputConnection con, PluginTask task) throws SQLException { if (task.getTable().isPresent()) { String actualTableName = normalizeTableNameCase(con, task.getTable().get()); @@ -426,8 +426,6 @@ public TaskReport run(TaskSource taskSource, LastRecordStore lastRecordStore = null; try (JdbcInputConnection con = newConnection(task)) { - con.before_load(); - List getters = newColumnGetters(con, task, querySchema, pageBuilder); try (BatchSelect cursor = con.newSelectCursor(builtQuery, getters, task.getFetchRows(), task.getSocketTimeout())) { while (true) { diff --git a/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java b/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java index f0569c9a..a27adecc 100644 --- a/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java +++ b/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java @@ -386,11 +386,4 @@ private Set getColumnNames(String tableName) throws SQLException return columnNamesBuilder.build(); } } - - public void before_load() - throws SQLException - { - - } - } diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLInputPlugin.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLInputPlugin.java index 5d72716c..73c16d0d 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLInputPlugin.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLInputPlugin.java @@ -12,10 +12,12 @@ import org.embulk.config.Config; import org.embulk.config.ConfigDefault; import org.embulk.input.jdbc.AbstractJdbcInputPlugin; +import org.embulk.input.jdbc.JdbcInputConnection; import org.embulk.input.jdbc.getter.ColumnGetterFactory; import org.embulk.input.mysql.MySQLInputConnection; import org.embulk.input.mysql.getter.MySQLColumnGetterFactory; import org.embulk.spi.PageBuilder; +import org.embulk.spi.Schema; import org.joda.time.DateTimeZone; public class MySQLInputPlugin @@ -163,4 +165,13 @@ private void loadTimeZoneMappings() } } } + + @Override + protected Schema setupTask(JdbcInputConnection con, PluginTask task) throws SQLException + { + MySQLInputConnection mySQLCon = (MySQLInputConnection)con; + mySQLCon.compareTimeZone(); + return super.setupTask(con,task); + } + } diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java index f49953f3..20735520 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java @@ -12,8 +12,7 @@ public class MySQLTimeZoneBuilder private static final int ONE_HOUR_SEC = 3600; private static final int ONE_MIN_SEC = 60; - public static TimeZone fromSystemTimeZone(Connection connection) - throws SQLException + public static TimeZone fromSystemTimeZone(Connection connection) throws SQLException { // // First, I used `@@system_time_zone`. but It return non Time Zone Abbreviations name on a specific platform. @@ -21,29 +20,34 @@ public static TimeZone fromSystemTimeZone(Connection connection) // String query = "select TIME_TO_SEC(timediff(now(),utc_timestamp()));"; Statement stmt = connection.createStatement(); - ResultSet rs = stmt.executeQuery(query); - if (rs.next()) { - int offset_seconds = rs.getInt(1); - return fromGMTOffsetSeconds(offset_seconds); - } - else { - // TODO Error check. - return null; + try { + ResultSet rs = stmt.executeQuery(query); + if (rs.next()) { + int offsetSeconds = rs.getInt(1); + return fromGMTOffsetSeconds(offsetSeconds); + } + else { + // TODO Error check. + return null; + } + + } finally { + stmt.close(); } } - public static TimeZone fromGMTOffsetSeconds(int offset_seconds) + private static TimeZone fromGMTOffsetSeconds(int offsetSeconds) { - if( offset_seconds == 0 ) { + if( offsetSeconds == 0 ) { return TimeZone.getTimeZone("UTC"); } - String sign = offset_seconds > 0 ? "+" : "-"; - int abs_offset_sec = Math.abs(offset_seconds); - int tz_hour = abs_offset_sec / ONE_HOUR_SEC; - int tz_min = abs_offset_sec % ONE_HOUR_SEC / ONE_MIN_SEC; - String tz_name = String.format(Locale.ENGLISH, "GMT%s%02d:%02d", sign, tz_hour, tz_min); - return TimeZone.getTimeZone(tz_name); + String sign = offsetSeconds > 0 ? "+" : "-"; + int absOffsetSec = Math.abs(offsetSeconds); + int tzHour = absOffsetSec / ONE_HOUR_SEC; + int tzMin = absOffsetSec % ONE_HOUR_SEC / ONE_MIN_SEC; + String tzName = String.format(Locale.ENGLISH, "GMT%s%02d:%02d", sign, tzHour, tzMin); + return TimeZone.getTimeZone(tzName); } } diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java index e5788874..e98fcf30 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java @@ -1,6 +1,7 @@ package org.embulk.input.mysql; import java.sql.Statement; +import java.util.Date; import java.util.List; import java.sql.Connection; import java.sql.PreparedStatement; @@ -63,15 +64,17 @@ public TimeZone getServerTimezoneTZ() return ((ConnectionImpl) connection).getServerTimezoneTZ(); } - @Override - public void before_load() - throws SQLException + public void compareTimeZone() throws SQLException { // TODO error check. - TimeZone svr_tz = MySQLTimeZoneBuilder.fromSystemTimeZone(connection); + TimeZone serverTimeZone = MySQLTimeZoneBuilder.fromSystemTimeZone(connection); + TimeZone clientTimeZone = TimeZone.getDefault(); + Date today = new Date(); + int clientOffset = clientTimeZone.getRawOffset(); - String usr_tz_name = System.getProperty("user.timezone"); - TimeZone usr_tz = TimeZone.getTimeZone(usr_tz_name); + if( clientTimeZone.inDaylightTime(today) ){ + clientOffset += clientTimeZone.getDSTSavings(); + } // // Compare offset only. Although I expect to return true, the following code return false, @@ -80,9 +83,10 @@ public void before_load() // TimeZone tz_gmt9 = TimeZone.getTimeZone("GMT+9"); // tz_jst.hasSameRules(tz_gmt9) // return false. // - if( svr_tz.getRawOffset() != usr_tz.getRawOffset() ) { + if( clientOffset != serverTimeZone.getRawOffset() ) { logger.warn(String.format(Locale.ENGLISH, - "The server timezone offset(%s) and client timezone(%s) has different timezone offset. The plugin will fetch wrong datetime values.",svr_tz.getID(),usr_tz_name)); + "The client timezone(%s) is different from the server timezone(%s). The plugin will fetch wrong datetime values.", + clientTimeZone.getID(),serverTimeZone.getID())); logger.warn(String.format(Locale.ENGLISH, "Use `options: { useLegacyDatetimeCode: false }`")); } From 14f82b0cc8a749d8ee828b29c34857a04bf2ed09 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sat, 13 May 2017 01:59:23 +0900 Subject: [PATCH 09/16] Make TimeZoneComparison class --- .../embulk/input/MySQLTimeZoneBuilder.java | 53 ---------- .../embulk/input/MySQLTimeZoneComparison.java | 97 +++++++++++++++++++ .../input/mysql/MySQLInputConnection.java | 32 +----- 3 files changed, 100 insertions(+), 82 deletions(-) delete mode 100644 embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java create mode 100644 embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java deleted file mode 100644 index 20735520..00000000 --- a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneBuilder.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.embulk.input; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Locale; -import java.util.TimeZone; - -public class MySQLTimeZoneBuilder -{ - private static final int ONE_HOUR_SEC = 3600; - private static final int ONE_MIN_SEC = 60; - - public static TimeZone fromSystemTimeZone(Connection connection) throws SQLException - { - // - // First, I used `@@system_time_zone`. but It return non Time Zone Abbreviations name on a specific platform. - // So, This method calculate GMT offset with query. - // - String query = "select TIME_TO_SEC(timediff(now(),utc_timestamp()));"; - Statement stmt = connection.createStatement(); - - try { - ResultSet rs = stmt.executeQuery(query); - if (rs.next()) { - int offsetSeconds = rs.getInt(1); - return fromGMTOffsetSeconds(offsetSeconds); - } - else { - // TODO Error check. - return null; - } - - } finally { - stmt.close(); - } - } - - private static TimeZone fromGMTOffsetSeconds(int offsetSeconds) - { - if( offsetSeconds == 0 ) { - return TimeZone.getTimeZone("UTC"); - } - - String sign = offsetSeconds > 0 ? "+" : "-"; - int absOffsetSec = Math.abs(offsetSeconds); - int tzHour = absOffsetSec / ONE_HOUR_SEC; - int tzMin = absOffsetSec % ONE_HOUR_SEC / ONE_MIN_SEC; - String tzName = String.format(Locale.ENGLISH, "GMT%s%02d:%02d", sign, tzHour, tzMin); - return TimeZone.getTimeZone(tzName); - } -} diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java new file mode 100644 index 00000000..ba4ce4cd --- /dev/null +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java @@ -0,0 +1,97 @@ +package org.embulk.input; + +import org.embulk.spi.Exec; +import org.slf4j.Logger; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Date; +import java.util.Locale; +import java.util.TimeZone; + +public class MySQLTimeZoneComparison +{ + private static final int ONE_HOUR_SEC = 3600; + private static final int ONE_MIN_SEC = 60; + + private Connection connection; + + private final Logger logger = Exec.getLogger(getClass()); + + public MySQLTimeZoneComparison(Connection connection) + { + this.connection = connection; + } + + public void compareTimeZone() throws SQLException + { + // TODO error check. + TimeZone serverTimeZone = getServerTimeZone(); + TimeZone clientTimeZone = TimeZone.getDefault(); + Date today = new Date(); + int clientOffset = clientTimeZone.getRawOffset(); + + if( clientTimeZone.inDaylightTime(today) ){ + clientOffset += clientTimeZone.getDSTSavings(); + } + + // + // Compare offset only. Although I expect to return true, the following code return false, + // + // TimeZone tz_jst = TimeZone.getTimeZone("JST"); + // TimeZone tz_gmt9 = TimeZone.getTimeZone("GMT+9"); + // tz_jst.hasSameRules(tz_gmt9) // return false. + // + if( clientOffset != serverTimeZone.getRawOffset() ) { + logger.warn(String.format(Locale.ENGLISH, + "The client timezone(%s) is different from the server timezone(%s). The plugin will fetch wrong datetime values.", + clientTimeZone.getID(),serverTimeZone.getID())); + logger.warn(String.format(Locale.ENGLISH, + "Use `options: { useLegacyDatetimeCode: false }`")); + } + logger.warn(String.format(Locale.ENGLISH,"The plugin will set `useLegacyDatetimeCode=false` by default in future.")); + + + } + + private TimeZone getServerTimeZone() throws SQLException + { + // + // First, I used `@@system_time_zone`. but It return non Time Zone Abbreviations name on a specific platform. + // So, This method calculate GMT offset with query. + // + String query = "select TIME_TO_SEC(timediff(now(),utc_timestamp()));"; + Statement stmt = connection.createStatement(); + + try { + ResultSet rs = stmt.executeQuery(query); + if (rs.next()) { + int offsetSeconds = rs.getInt(1); + return fromGMTOffsetSeconds(offsetSeconds); + } + else { + // TODO Error check. + return null; + } + + } finally { + stmt.close(); + } + } + + private TimeZone fromGMTOffsetSeconds(int offsetSeconds) + { + if( offsetSeconds == 0 ) { + return TimeZone.getTimeZone("UTC"); + } + + String sign = offsetSeconds > 0 ? "+" : "-"; + int absOffsetSec = Math.abs(offsetSeconds); + int tzHour = absOffsetSec / ONE_HOUR_SEC; + int tzMin = absOffsetSec % ONE_HOUR_SEC / ONE_MIN_SEC; + String tzName = String.format(Locale.ENGLISH, "GMT%s%02d:%02d", sign, tzHour, tzMin); + return TimeZone.getTimeZone(tzName); + } +} diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java index e98fcf30..c77cb6ab 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java @@ -1,18 +1,15 @@ package org.embulk.input.mysql; -import java.sql.Statement; -import java.util.Date; import java.util.List; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.ResultSet; -import java.util.Locale; import java.util.TimeZone; import com.mysql.jdbc.ConnectionImpl; import com.mysql.jdbc.ConnectionProperties; -import org.embulk.input.MySQLTimeZoneBuilder; +import org.embulk.input.MySQLTimeZoneComparison; import org.embulk.input.jdbc.JdbcInputConnection; import org.embulk.input.jdbc.JdbcLiteral; import org.embulk.input.jdbc.getter.ColumnGetter; @@ -66,31 +63,8 @@ public TimeZone getServerTimezoneTZ() public void compareTimeZone() throws SQLException { - // TODO error check. - TimeZone serverTimeZone = MySQLTimeZoneBuilder.fromSystemTimeZone(connection); - TimeZone clientTimeZone = TimeZone.getDefault(); - Date today = new Date(); - int clientOffset = clientTimeZone.getRawOffset(); - - if( clientTimeZone.inDaylightTime(today) ){ - clientOffset += clientTimeZone.getDSTSavings(); - } - - // - // Compare offset only. Although I expect to return true, the following code return false, - // - // TimeZone tz_jst = TimeZone.getTimeZone("JST"); - // TimeZone tz_gmt9 = TimeZone.getTimeZone("GMT+9"); - // tz_jst.hasSameRules(tz_gmt9) // return false. - // - if( clientOffset != serverTimeZone.getRawOffset() ) { - logger.warn(String.format(Locale.ENGLISH, - "The client timezone(%s) is different from the server timezone(%s). The plugin will fetch wrong datetime values.", - clientTimeZone.getID(),serverTimeZone.getID())); - logger.warn(String.format(Locale.ENGLISH, - "Use `options: { useLegacyDatetimeCode: false }`")); - } - logger.warn(String.format(Locale.ENGLISH,"The plugin will set `useLegacyDatetimeCode=false` by default in future.")); + MySQLTimeZoneComparison timeZoneComparison = new MySQLTimeZoneComparison(connection); + timeZoneComparison.compareTimeZone(); } } From 275dbf3cece95f3b6d7241b33b42ebe8b2997c88 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 17 May 2017 15:06:27 +0900 Subject: [PATCH 10/16] Add error check and change log messages --- .../embulk/input/MySQLTimeZoneComparison.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java index ba4ce4cd..54bea4a1 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java @@ -27,8 +27,18 @@ public MySQLTimeZoneComparison(Connection connection) public void compareTimeZone() throws SQLException { - // TODO error check. - TimeZone serverTimeZone = getServerTimeZone(); + TimeZone serverTimeZone = null; + try { + serverTimeZone = getServerTimeZone(); + } catch (SQLException ex) { + logger.error(String.format(Locale.ENGLISH,"SQLException raised %s",ex.toString())); + } + + if( serverTimeZone == null ){ + logger.warn("Can't get server TimeZone"); + return; + } + TimeZone clientTimeZone = TimeZone.getDefault(); Date today = new Date(); int clientOffset = clientTimeZone.getRawOffset(); @@ -49,7 +59,9 @@ public void compareTimeZone() throws SQLException "The client timezone(%s) is different from the server timezone(%s). The plugin will fetch wrong datetime values.", clientTimeZone.getID(),serverTimeZone.getID())); logger.warn(String.format(Locale.ENGLISH, - "Use `options: { useLegacyDatetimeCode: false }`")); + "Use `You may need to set options useLegacyDatetimeCode and serverTimeZone`")); + logger.warn(String.format(Locale.ENGLISH, + "Ex. `options: { useLegacyDatetimeCode: false, serverTimeZone: UTC }`")); } logger.warn(String.format(Locale.ENGLISH,"The plugin will set `useLegacyDatetimeCode=false` by default in future.")); @@ -72,7 +84,6 @@ private TimeZone getServerTimeZone() throws SQLException return fromGMTOffsetSeconds(offsetSeconds); } else { - // TODO Error check. return null; } From 9a03bcfd70c2ac797c0525c0bbf5af564a4a00f7 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 17 May 2017 15:13:56 +0900 Subject: [PATCH 11/16] Change error messages --- .../src/main/java/org/embulk/input/MySQLTimeZoneComparison.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java index 54bea4a1..ddd4277d 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java @@ -59,7 +59,7 @@ public void compareTimeZone() throws SQLException "The client timezone(%s) is different from the server timezone(%s). The plugin will fetch wrong datetime values.", clientTimeZone.getID(),serverTimeZone.getID())); logger.warn(String.format(Locale.ENGLISH, - "Use `You may need to set options useLegacyDatetimeCode and serverTimeZone`")); + "Use You may need to set options `useLegacyDatetimeCode` and `serverTimeZone`")); logger.warn(String.format(Locale.ENGLISH, "Ex. `options: { useLegacyDatetimeCode: false, serverTimeZone: UTC }`")); } From eb2ae73e5ee2bd4248fa5318d9c433e91dfc7f35 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 17 May 2017 15:15:51 +0900 Subject: [PATCH 12/16] Change error message. add period --- .../src/main/java/org/embulk/input/MySQLTimeZoneComparison.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java index ddd4277d..28538ef5 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java @@ -35,7 +35,7 @@ public void compareTimeZone() throws SQLException } if( serverTimeZone == null ){ - logger.warn("Can't get server TimeZone"); + logger.warn("Can't get server TimeZone."); return; } From c333c7d52097d677fedbeb938427721239472aba Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 17 May 2017 15:22:11 +0900 Subject: [PATCH 13/16] Reformat code --- .../embulk/input/MySQLTimeZoneComparison.java | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java index 28538ef5..3fcfc8b2 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java @@ -25,16 +25,18 @@ public MySQLTimeZoneComparison(Connection connection) this.connection = connection; } - public void compareTimeZone() throws SQLException + public void compareTimeZone() + throws SQLException { TimeZone serverTimeZone = null; try { serverTimeZone = getServerTimeZone(); - } catch (SQLException ex) { - logger.error(String.format(Locale.ENGLISH,"SQLException raised %s",ex.toString())); + } + catch (SQLException ex) { + logger.error(String.format(Locale.ENGLISH, "SQLException raised %s", ex.toString())); } - if( serverTimeZone == null ){ + if (serverTimeZone == null) { logger.warn("Can't get server TimeZone."); return; } @@ -43,8 +45,8 @@ public void compareTimeZone() throws SQLException Date today = new Date(); int clientOffset = clientTimeZone.getRawOffset(); - if( clientTimeZone.inDaylightTime(today) ){ - clientOffset += clientTimeZone.getDSTSavings(); + if (clientTimeZone.inDaylightTime(today)) { + clientOffset += clientTimeZone.getDSTSavings(); } // @@ -54,21 +56,20 @@ public void compareTimeZone() throws SQLException // TimeZone tz_gmt9 = TimeZone.getTimeZone("GMT+9"); // tz_jst.hasSameRules(tz_gmt9) // return false. // - if( clientOffset != serverTimeZone.getRawOffset() ) { + if (clientOffset != serverTimeZone.getRawOffset()) { logger.warn(String.format(Locale.ENGLISH, "The client timezone(%s) is different from the server timezone(%s). The plugin will fetch wrong datetime values.", - clientTimeZone.getID(),serverTimeZone.getID())); + clientTimeZone.getID(), serverTimeZone.getID())); logger.warn(String.format(Locale.ENGLISH, "Use You may need to set options `useLegacyDatetimeCode` and `serverTimeZone`")); logger.warn(String.format(Locale.ENGLISH, "Ex. `options: { useLegacyDatetimeCode: false, serverTimeZone: UTC }`")); } - logger.warn(String.format(Locale.ENGLISH,"The plugin will set `useLegacyDatetimeCode=false` by default in future.")); - - + logger.warn(String.format(Locale.ENGLISH, "The plugin will set `useLegacyDatetimeCode=false` by default in future.")); } - private TimeZone getServerTimeZone() throws SQLException + private TimeZone getServerTimeZone() + throws SQLException { // // First, I used `@@system_time_zone`. but It return non Time Zone Abbreviations name on a specific platform. @@ -86,15 +87,15 @@ private TimeZone getServerTimeZone() throws SQLException else { return null; } - - } finally { + } + finally { stmt.close(); } } private TimeZone fromGMTOffsetSeconds(int offsetSeconds) { - if( offsetSeconds == 0 ) { + if (offsetSeconds == 0) { return TimeZone.getTimeZone("UTC"); } From 21adf05812c021a83945bf51a470439f69264bda Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Fri, 19 May 2017 17:28:03 +0900 Subject: [PATCH 14/16] Change log level and avoid null check if exception caught. --- .../embulk/input/MySQLTimeZoneComparison.java | 60 +++++++++---------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java index 3fcfc8b2..1444239c 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java @@ -31,41 +31,39 @@ public void compareTimeZone() TimeZone serverTimeZone = null; try { serverTimeZone = getServerTimeZone(); - } - catch (SQLException ex) { - logger.error(String.format(Locale.ENGLISH, "SQLException raised %s", ex.toString())); - } - - if (serverTimeZone == null) { - logger.warn("Can't get server TimeZone."); - return; - } + if (serverTimeZone == null) { + logger.warn("Can't get server TimeZone."); + return; + } - TimeZone clientTimeZone = TimeZone.getDefault(); - Date today = new Date(); - int clientOffset = clientTimeZone.getRawOffset(); + TimeZone clientTimeZone = TimeZone.getDefault(); + Date today = new Date(); + int clientOffset = clientTimeZone.getRawOffset(); - if (clientTimeZone.inDaylightTime(today)) { - clientOffset += clientTimeZone.getDSTSavings(); - } + if (clientTimeZone.inDaylightTime(today)) { + clientOffset += clientTimeZone.getDSTSavings(); + } - // - // Compare offset only. Although I expect to return true, the following code return false, - // - // TimeZone tz_jst = TimeZone.getTimeZone("JST"); - // TimeZone tz_gmt9 = TimeZone.getTimeZone("GMT+9"); - // tz_jst.hasSameRules(tz_gmt9) // return false. - // - if (clientOffset != serverTimeZone.getRawOffset()) { - logger.warn(String.format(Locale.ENGLISH, - "The client timezone(%s) is different from the server timezone(%s). The plugin will fetch wrong datetime values.", - clientTimeZone.getID(), serverTimeZone.getID())); - logger.warn(String.format(Locale.ENGLISH, - "Use You may need to set options `useLegacyDatetimeCode` and `serverTimeZone`")); - logger.warn(String.format(Locale.ENGLISH, - "Ex. `options: { useLegacyDatetimeCode: false, serverTimeZone: UTC }`")); + // + // Compare offset only. Although I expect to return true, the following code return false, + // + // TimeZone tz_jst = TimeZone.getTimeZone("JST"); + // TimeZone tz_gmt9 = TimeZone.getTimeZone("GMT+9"); + // tz_jst.hasSameRules(tz_gmt9) // return false. + // + if (clientOffset != serverTimeZone.getRawOffset()) { + logger.warn(String.format(Locale.ENGLISH, + "The client timezone(%s) is different from the server timezone(%s). The plugin will fetch wrong datetime values.", + clientTimeZone.getID(), serverTimeZone.getID())); + logger.warn(String.format(Locale.ENGLISH, + "You may need to set options `useLegacyDatetimeCode` and `serverTimeZone`")); + logger.warn(String.format(Locale.ENGLISH, + "Example: `options: { useLegacyDatetimeCode: false, serverTimeZone: UTC }`")); + } + logger.warn(String.format(Locale.ENGLISH, "The plugin will set `useLegacyDatetimeCode=false` by default in future.")); } + catch (SQLException ex) { + logger.warn(String.format(Locale.ENGLISH, "SQLException raised %s", ex.toString())); } - logger.warn(String.format(Locale.ENGLISH, "The plugin will set `useLegacyDatetimeCode=false` by default in future.")); } private TimeZone getServerTimeZone() From 7ca3a9bb06866b6c010449985b402deefb37cd62 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Fri, 19 May 2017 18:12:13 +0900 Subject: [PATCH 15/16] throw SQLException if timezone comparison query fail. --- .../embulk/input/MySQLTimeZoneComparison.java | 61 +++++++++---------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java index 1444239c..41e47ac4 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java @@ -31,39 +31,37 @@ public void compareTimeZone() TimeZone serverTimeZone = null; try { serverTimeZone = getServerTimeZone(); - if (serverTimeZone == null) { - logger.warn("Can't get server TimeZone."); - return; - } + } + catch (SQLException ex) { + logger.warn("Can't get server TimeZone."); + logger.warn(String.format(Locale.ENGLISH, "SQLException raised %s", ex.toString())); + } - TimeZone clientTimeZone = TimeZone.getDefault(); - Date today = new Date(); - int clientOffset = clientTimeZone.getRawOffset(); + TimeZone clientTimeZone = TimeZone.getDefault(); + Date today = new Date(); + int clientOffset = clientTimeZone.getRawOffset(); - if (clientTimeZone.inDaylightTime(today)) { - clientOffset += clientTimeZone.getDSTSavings(); - } + if (clientTimeZone.inDaylightTime(today)) { + clientOffset += clientTimeZone.getDSTSavings(); + } - // - // Compare offset only. Although I expect to return true, the following code return false, - // - // TimeZone tz_jst = TimeZone.getTimeZone("JST"); - // TimeZone tz_gmt9 = TimeZone.getTimeZone("GMT+9"); - // tz_jst.hasSameRules(tz_gmt9) // return false. - // - if (clientOffset != serverTimeZone.getRawOffset()) { - logger.warn(String.format(Locale.ENGLISH, - "The client timezone(%s) is different from the server timezone(%s). The plugin will fetch wrong datetime values.", - clientTimeZone.getID(), serverTimeZone.getID())); - logger.warn(String.format(Locale.ENGLISH, - "You may need to set options `useLegacyDatetimeCode` and `serverTimeZone`")); - logger.warn(String.format(Locale.ENGLISH, - "Example: `options: { useLegacyDatetimeCode: false, serverTimeZone: UTC }`")); - } - logger.warn(String.format(Locale.ENGLISH, "The plugin will set `useLegacyDatetimeCode=false` by default in future.")); } - catch (SQLException ex) { - logger.warn(String.format(Locale.ENGLISH, "SQLException raised %s", ex.toString())); + // + // Compare offset only. Although I expect to return true, the following code return false, + // + // TimeZone tz_jst = TimeZone.getTimeZone("JST"); + // TimeZone tz_gmt9 = TimeZone.getTimeZone("GMT+9"); + // tz_jst.hasSameRules(tz_gmt9) // return false. + // + if (clientOffset != serverTimeZone.getRawOffset()) { + logger.warn(String.format(Locale.ENGLISH, + "The client timezone(%s) is different from the server timezone(%s). The plugin will fetch wrong datetime values.", + clientTimeZone.getID(), serverTimeZone.getID())); + logger.warn(String.format(Locale.ENGLISH, + "You may need to set options `useLegacyDatetimeCode` and `serverTimeZone`")); + logger.warn(String.format(Locale.ENGLISH, + "Example. `options: { useLegacyDatetimeCode: false, serverTimeZone: UTC }`")); } + logger.warn(String.format(Locale.ENGLISH, "The plugin will set `useLegacyDatetimeCode=false` by default in future.")); } private TimeZone getServerTimeZone() @@ -82,9 +80,8 @@ private TimeZone getServerTimeZone() int offsetSeconds = rs.getInt(1); return fromGMTOffsetSeconds(offsetSeconds); } - else { - return null; - } + throw new SQLException(String.format(Locale.ENGLISH, + "The timezone comparison query(%s) doesn't return the result.",query)); } finally { stmt.close(); From dd2c693877a4ac754cfa51ef7b0403387aeb54ca Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 24 May 2017 19:16:07 +0900 Subject: [PATCH 16/16] Fix typo --- .../main/java/org/embulk/input/MySQLTimeZoneComparison.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java index 41e47ac4..95829c24 100644 --- a/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java +++ b/embulk-input-mysql/src/main/java/org/embulk/input/MySQLTimeZoneComparison.java @@ -57,9 +57,9 @@ public void compareTimeZone() "The client timezone(%s) is different from the server timezone(%s). The plugin will fetch wrong datetime values.", clientTimeZone.getID(), serverTimeZone.getID())); logger.warn(String.format(Locale.ENGLISH, - "You may need to set options `useLegacyDatetimeCode` and `serverTimeZone`")); + "You may need to set options `useLegacyDatetimeCode` and `serverTimezone`")); logger.warn(String.format(Locale.ENGLISH, - "Example. `options: { useLegacyDatetimeCode: false, serverTimeZone: UTC }`")); + "Example. `options: { useLegacyDatetimeCode: false, serverTimezone: UTC }`")); } logger.warn(String.format(Locale.ENGLISH, "The plugin will set `useLegacyDatetimeCode=false` by default in future.")); }