注解完成部分语法和测试用例，sql和catlet语法还需完善

Author: ishotoli

minimal_sql_tokens.txt

@@ -1,4 +1,5 @@
 AS
+ID
 IF
 ON
 FOR
@@ -11,39 +12,69 @@ DESC
 FROM
 INTO
 JOIN
+KILL
 LOCK
 LEFT
+NAME
 SHOW
+SLOW
+TIME
+USER
 ALTER
+CLEAR
+CACHE
 GROUP
 GRANT
+INDEX
 LIMIT
 MYCAT
 ORDER
 RIGHT
+ROUTE
 TABLE
 UNION
 WHERE
 CATLET
+CONFIG
 CREATE
-EXISTS
-UPDATE
+DETAIL
 DELETE
-SELECT
+EXISTS
 INSERT
 IGNORE
-SCHEMA
+ONLINE
+RELOAD
 RENAME
 REVOKE
+ROUTER
+SELECT
+SERVER
+SCHEMA
+SWITCH
+UPDATE
+BACKEND
+BALANCE
+CURRENT
 DB_TYPE
+DELAYED
+EXECUTE
 EXPLAIN
-BALANCE
+OFFLINE
 REPLACE
-DELAYED
+SESSION
+STARTUP
+VERSION
+DATABASE
 DATANODE
 DESCRIBE
+ROLLBACK
 TRUNCATE
+HEARTBEAT
 PROCEDURE
+PROCESSOR
+CONNECTION
+DATASOURCE
+THREADPOOL
 LOW_PRIORITY
 HIGH_PRIORITY
 

sql_tokens.txt

@@ -24,7 +24,9 @@ AUTOEXTEND_SIZE
 AUTO_INCREMENT
 AVG_ROW_LENGTH
 AVG
+BACKEND
 BACKUP
+BALANCE
 BEFORE
 BEGIN
 BETWEEN
@@ -51,6 +53,7 @@ CASCADED
 CASE
 CAST
 CATALOG_NAME
+CATLET
 CHAIN
 CHANGE
 CHANGED
@@ -62,6 +65,7 @@ CHECKSUM
 CHECK
 CIPHER
 CLASS_ORIGIN
+CLEAR
 CLIENT
 CLOSE
 COALESCE
@@ -81,6 +85,7 @@ COMPRESSED
 COMPRESSION
 CONCURRENT
 CONDITION
+CONFIG
 CONNECTION
 CONSISTENT
 CONSTRAINT
@@ -110,6 +115,8 @@ DATABASE
 DATABASES
 DATAFILE
 DATA
+DATANODE
+DATASOURCE
 DATETIME
 DATE_ADD
 DATE_ADD_INTERVAL
@@ -122,6 +129,7 @@ DAY_MICROSECOND
 DAY_MINUTE
 DAY_SECOND
 DAY
+DB_TYPE
 DEALLOCATE
 DEC
 DECIMAL_NUM
@@ -136,6 +144,7 @@ DELETE
 DESC
 DESCRIBE
 DES_KEY_FILE
+DETAIL
 DETERMINISTIC
 DIAGNOSTICS
 DIRECTORY
@@ -196,9 +205,9 @@ FLOAT8
 FLOAT
 FLUSH
 FOLLOWS
+FOR
 FORCE
 FOREIGN
-FOR
 FORMAT
 FOUND
 FRAC_SECOND
@@ -221,6 +230,7 @@ GROUP_CONCAT
 HANDLER
 HASH
 HAVING
+HEARTBEAT
 HELP
 HIGH_PRIORITY
 HOST
@@ -229,6 +239,7 @@ HOUR_MICROSECOND
 HOUR_MINUTE
 HOUR_SECOND
 HOUR
+ID
 IDENTIFIED
 IF
 IGNORE
@@ -353,6 +364,7 @@ MULTILINESTRING
 MULTIPOINT
 MULTIPOLYGON
 MUTEX
+MYCAT
 MYSQL_ERRNO
 NAMES
 NAME
@@ -423,6 +435,7 @@ PRIMARY
 PRIVILEGES
 PROCEDURE
 PROCESS
+PROCESSOR
 PROCESSLIST
 PROFILE
 PROFILES
@@ -481,6 +494,8 @@ RIGHT
 RLIKE
 ROLLBACK
 ROLLUP
+ROUTE
+ROUTER
 ROUTINE
 ROWS
 ROW_COUNT
@@ -542,6 +557,7 @@ STACKED
 STARTING
 STARTS
 START
+STARTUP
 STATS_AUTO_RECALC
 STATS_PERSISTENT
 STATS_SAMPLE_PAGES
@@ -581,6 +597,7 @@ TERMINATED
 TEXT
 THAN
 THEN
+THREADPOOL
 TIMESTAMP
 TIMESTAMP_ADD
 TIMESTAMP_DIFF

src/main/java/io/mycat/IntTokenHash.java

@@ -4,55 +4,88 @@
  * Created by Kaiz on 2017/3/4.
  */
 public class IntTokenHash {
+    public static final int ANNOTATION_BALANCE= 0x00140001;
+    public static final int ANNOTATION_START = 0x00150002;
+    public static final int ANNOTATION_END   = 0x00160002;
+
+    //generate by MatchMethodGenerator.GenerateSqlTokenHash
     public static final int AS               = 0x46580002;
+    public static final int ID               = 0x47670002;
     public static final int IF               = 0x47650002;
     public static final int ON               = 0x409e0002;
-
     public static final int FOR              = 0x576e0003;
     public static final int NOT              = 0x68b00003;
     public static final int SQL              = 0x53820003;
     public static final int USE              = 0x0f840003;
-
+    public static final int CALL             = 0x25d00004;
     public static final int DROP             = 0xba5d0004;
+    public static final int DESC             = 0xc08f0004;
     public static final int FROM             = 0x992e0004;
     public static final int INTO             = 0xb2e70004;
     public static final int JOIN             = 0xf9700004;
+    public static final int KILL             = 0x5f7f0004;
     public static final int LOCK             = 0xed330004;
     public static final int LEFT             = 0xf4e50004;
+    public static final int NAME             = 0x33200004;
     public static final int SHOW             = 0xb0300004;
-
+    public static final int SLOW             = 0xa3340004;
+    public static final int TIME             = 0x5f050004;
+    public static final int USER             = 0x7bfa0004;
     public static final int ALTER            = 0xb2db0005;
+    public static final int CLEAR            = 0x19bf0005;
+    public static final int CACHE            = 0xf61c0005;
     public static final int GROUP            = 0x43f50005;
+    public static final int GRANT            = 0x29c20005;
+    public static final int INDEX            = 0xfcff0005;
     public static final int LIMIT            = 0x450a0005;
     public static final int MYCAT            = 0xdce50005;
     public static final int ORDER            = 0x88da0005;
     public static final int RIGHT            = 0xec610005;
+    public static final int ROUTE            = 0x82ef0005;
     public static final int TABLE            = 0x03780005;
     public static final int UNION            = 0xcf9d0005;
     public static final int WHERE            = 0x75950005;
-
     public static final int CATLET           = 0xda600006;
+    public static final int CONFIG           = 0xebd60006;
     public static final int CREATE           = 0xdde00006;
-    public static final int EXISTS           = 0x2ed70006;
-    public static final int UPDATE           = 0x62840006;
+    public static final int DETAIL           = 0x47690006;
     public static final int DELETE           = 0x33e50006;
-    public static final int SELECT           = 0x02a10006;
+    public static final int EXISTS           = 0x2ed70006;
     public static final int INSERT           = 0xa3b10006;
     public static final int IGNORE           = 0x6c360006;
+    public static final int ONLINE           = 0x05bc0006;
+    public static final int RELOAD           = 0xd7200006;
+    public static final int RENAME           = 0xe6c60006;
+    public static final int REVOKE           = 0x5cc20006;
+    public static final int ROUTER           = 0x3c570006;
+    public static final int SELECT           = 0x02a10006;
+    public static final int SERVER           = 0x31fa0006;
     public static final int SCHEMA           = 0x0c490006;
-
-    public static final int DB_TYPE          = 0xac610007;
+    public static final int SWITCH           = 0xe0500006;
+    public static final int UPDATE           = 0x62840006;
+    public static final int BACKEND          = 0x9c0b0007;
     public static final int BALANCE          = 0x8d070007;
-    public static final int REPLACE          = 0xa88c0007;
+    public static final int CURRENT          = 0x6af60007;
+    public static final int DB_TYPE          = 0xac610007;
     public static final int DELAYED          = 0x56380007;
-
+    public static final int EXECUTE          = 0x304b0007;
+    public static final int EXPLAIN          = 0xe41f0007;
+    public static final int OFFLINE          = 0x9ead0007;
+    public static final int REPLACE          = 0xa88c0007;
+    public static final int SESSION          = 0xf2080007;
+    public static final int STARTUP          = 0x16770007;
+    public static final int VERSION          = 0x66380007;
+    public static final int DATABASE         = 0xbbcc0008;
     public static final int DATANODE         = 0x045d0008;
+    public static final int DESCRIBE         = 0x8ad30008;
+    public static final int ROLLBACK         = 0xe7390008;
     public static final int TRUNCATE         = 0xcb5d0008;
-
+    public static final int HEARTBEAT        = 0xabb70009;
     public static final int PROCEDURE        = 0x11600009;
-
+    public static final int PROCESSOR        = 0xfdbc0009;
+    public static final int CONNECTION       = 0x307d000a;
+    public static final int DATASOURCE       = 0x0b35000a;
+    public static final int THREADPOOL       = 0x5a7a000a;
     public static final int LOW_PRIORITY     = 0xb805000c;
-
     public static final int HIGH_PRIORITY    = 0x7c6f000d;
-
 }

src/main/java/io/mycat/MatchMethodGenerator.java

@@ -131,6 +131,7 @@ static void initShrinkCharTbl () {
         shrinkCharTbl[0] = 1;//从 $ 开始计算
         IntStream.rangeClosed('0', '9').forEach(c -> shrinkCharTbl[c-'$'] = (byte)(c-'0'+2));
         IntStream.rangeClosed('A', 'Z').forEach(c -> shrinkCharTbl[c-'$'] = (byte)(c-'A'+12));
+        IntStream.rangeClosed('a', 'z').forEach(c -> shrinkCharTbl[c-'$'] = (byte)(c-'a'+12));
         shrinkCharTbl['_'-'$'] = (byte)38;
     }
 
@@ -327,21 +328,36 @@ static void skipXXXTokenGenerator() {
             System.out.format("final void skip%sToken() {\npos+=%d;\n}%n", keyword, keyword.length());
         });
     }
-    static void GenerateSqlTokenHash(String fileName) {
+    static void GenerateLongTokenHash(String fileName) {
         initShrinkCharTbl();
         try {
             Files.lines(Paths.get(fileName))
                     .filter(x -> x.length()>0)
                     .forEach(x -> {
-                System.out.format("    public static final int %-16s = 0x%04x%04x;%n", x, genHash2(x.toCharArray()) & 0xFFFF, x.length());
-//                System.out.format("    public static final int %s = 0x%x;%n", x, genHash2(x.toCharArray()));
+//                System.out.format("    public static final int %-16s = 0x%04x%04x;%n", x, genHash2(x.toCharArray()) & 0xFFFF, x.length());
+                System.out.format("    public static final long %-12s = 0x%xL;%n", x, genHash(x.toCharArray()));
             });
 //            System.out.println("conflict count : "+count);
         } catch (IOException e) {
             e.printStackTrace();
         }
     }
 
+    static void GenerateIntTokenHash(String fileName) {
+        initShrinkCharTbl();
+        try {
+            Files.lines(Paths.get(fileName))
+                    .filter(x -> x.length()>0)
+                    .forEach(x -> {
+                System.out.format("    public static final int %-16s = 0x%04x%04x;%n", x, genHash2(x.toCharArray()) & 0xFFFF, x.length());
+//                        System.out.format("    public static final long %-12s = 0x%x;%n", x, genHash(x.toCharArray()));
+                    });
+//            System.out.println("conflict count : "+count);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
     public static void main(String[] args) {
         //isXXXTokenGenerator();
         //skipXXXTokenGenerator();
@@ -350,6 +366,11 @@ public static void main(String[] args) {
 //        sqlKeyHastTest("minimal_sql_tokens.txt", s -> genHash(s.toCharArray()), 0x3FL);
 //        run();
 //        test1();
-        GenerateSqlTokenHash("minimal_sql_tokens.txt");
+//        GenerateIntTokenHash("minimal_sql_tokens.txt");
+//        GenerateLongTokenHash("sql_tokens.txt");
+        initShrinkCharTbl();
+        System.out.format("0x%xL;%n", genHash("dn1".toCharArray()));
+
+
     }
 }