HPCC4J-657 DFSClient: Reading indices with blobs fails

jpmcmu · jpmcmu · commit 31344aaee949 · 2024-10-23T20:00:40.000-04:00
- Updated RecordDefinitionTranslator to maintain blob information
- Added isBlob property to FieldDef
- Added translation test for indices with blobs
- Added index read test for indices with blobs
- Updated data generation to create an index with blobs

Signed-off-by: James McMullan James.McMullan@lexisnexis.com
diff --git a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java
@@ -35,6 +35,7 @@ public class FieldDef implements Serializable
     private long              len                = 0;
     private boolean           fixedLength        = false;
     private boolean           isUnsigned         = false;
+    private boolean           isBlob             = false;
     private int               additionalFlags    = 0;
 
     /**
@@ -59,6 +60,7 @@ public FieldDef(FieldDef rhs)
         this.len = rhs.len;
         this.fixedLength = rhs.fixedLength;
         this.isUnsigned = rhs.isUnsigned;
+        this.isBlob = rhs.isBlob;
         this.additionalFlags = rhs.additionalFlags;
     }
 
@@ -330,6 +332,24 @@ public boolean isBiased()
         return isNonStandardInt();
     }
 
+    /**
+     * Is the field stored in a blob?
+     *
+     * @return true when blob
+     */
+    public boolean isBlob()
+    {
+        return this.isBlob;
+    }
+
+    /**
+     * Sets the blob flag.
+     */
+    public void setIsBlob(boolean blob)
+    {
+        this.isBlob = blob;
+    }
+
     /**
      *
      *
diff --git a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java
@@ -32,6 +32,7 @@ public class RecordDefinitionTranslator
     private static final String NAME_KEY              = "name";
     private static final String TYPE_KEY              = "type";
     private static final String CHILD_KEY             = "child";
+    private static final String XPATH_KEY             = "xpath";
     private static final String FLAGS_KEY             = "flags";
 
     private static final String ESP_TYPE_NAME_PREFIX = "ty";
@@ -48,6 +49,7 @@ public class RecordDefinitionTranslator
     final private static int    type_keyedint         = 10;         // Convert to integer
     final private static int    type_record           = 13;
     final private static int    type_varstring        = 14;
+    final private static int    type_blob             = 15;
     final private static int    type_data             = 16;
     final private static int    type_table            = 20;
     final private static int    type_set              = 21;
@@ -264,28 +266,32 @@ public static String toECLRecord(FieldDef field) throws Exception
      */
     private static String getEClTypeDefinition(FieldDef field, HashMap<String, String> recordDefinitionMap) throws Exception
     {
+        String type = "";
         switch (field.getFieldType())
         {
             case SET:
             {
-                return "SET OF " + getEClTypeDefinition(field.getDef(0), recordDefinitionMap);
+                type = "SET OF " + getEClTypeDefinition(field.getDef(0), recordDefinitionMap);
+                break;
             }
             case DATASET:
             {
-                return "DATASET(" + getEClTypeDefinition(field.getDef(0), recordDefinitionMap) + ")";
+                type = "DATASET(" + getEClTypeDefinition(field.getDef(0), recordDefinitionMap) + ")";
+                break;
             }
             case BINARY:
             {
+                type = "DATA";
                 if (field.isFixed())
                 {
-                    return "DATA" + field.getDataLen();
+                    type += field.getDataLen();
                 }
-
-                return "DATA";
+                break;
             }
             case BOOLEAN:
             {
-                return "BOOLEAN";
+                type = "BOOLEAN";
+                break;
             }
             case INTEGER:
             {
@@ -300,7 +306,8 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
                     throw new Exception("Error: Unsupported integer size: " + field.getDataLen() + " must 1-8.");
                 }
 
-                return root + field.getDataLen();
+                type = root + field.getDataLen();
+                break;
             }
             case FILEPOS:
             {
@@ -314,7 +321,8 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
                     throw new Exception("Error: Unsupported filepos size: " + field.getDataLen() + " must be 8.");
                 }
 
-                return "UNSIGNED8";
+                type = "UNSIGNED8";
+                break;
             }
             case DECIMAL:
             {
@@ -323,28 +331,33 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
                 {
                     root = "U" + root;
                 }
-                return root + field.getPrecision() + "_" + field.getScale();
+                type = root + field.getPrecision() + "_" + field.getScale();
+                break;
             }
             case REAL:
             {
                 if (field.getDataLen() == 4)
                 {
-                    return "REAL4";
+                    type = "REAL4";
                 }
                 else if (field.getDataLen() == 8)
                 {
-                    return "REAL8";
+                    type = "REAL8";
                 }
+                else
+                {
+                    throw new Exception("Error: Unsupported real size: " + field.getDataLen() + " must 4 or 8.");
+                }
+                break;
 
-                throw new Exception("Error: Unsupported real size: " + field.getDataLen() + " must 4 or 8.");
             }
             case CHAR:
             {
-                return "STRING1";
+                type = "STRING1";
+                break;
             }
             case STRING:
             {
-                String type = "";
                 HpccSrcType srcType = field.getSourceType();
                 if (srcType == HpccSrcType.SINGLE_BYTE_CHAR)
                 {
@@ -371,11 +384,10 @@ else if (srcType == HpccSrcType.QSTRING)
                 {
                     type += field.getDataLen();
                 }
-                return type;
+                break;
             }
             case VAR_STRING:
             {
-                String type = "";
                 HpccSrcType srcType = field.getSourceType();
                 if (srcType == HpccSrcType.SINGLE_BYTE_CHAR)
                 {
@@ -394,7 +406,7 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
                 {
                     type += field.getDataLen();
                 }
-                return type;
+                break;
             }
             case RECORD:
             {
@@ -416,13 +428,21 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
                 String recordDefnName = "##" + hash + "##";
 
                 recordDefinitionMap.put(recordDefnName, definition);
-                return recordDefnName;
+                type = recordDefnName;
+                break;
             }
             default:
             {
                 throw new Exception("Unable to generate ECL unknown field type: " + field.getFieldType().description());
             }
         }
+
+        if (field.isBlob())
+        {
+            type += " {blob}";
+        }
+
+        return type;
     }
 
     /**
@@ -648,7 +668,7 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
      */
     private static int getTypeHash(FieldDef field) throws Exception
     {
-        int numHashComponents = 4 + field.getNumDefs();
+        int numHashComponents = 5 + field.getNumDefs();
         if (field.getFieldType() == FieldType.DECIMAL)
         {
             numHashComponents += 2;
@@ -659,8 +679,9 @@ private static int getTypeHash(FieldDef field) throws Exception
         hashComponents[1] = field.getDataLen();
         hashComponents[2] = field.getSourceType().ordinal();
         hashComponents[3] = field.getAdditionalFlags();
+        hashComponents[4] = field.isBlob() ? 1 : 0;
 
-        int hashCompIndex = 4;
+        int hashCompIndex = 5;
         for (int i = 0; i < field.getNumDefs(); i++, hashCompIndex++)
         {
             hashComponents[hashCompIndex] = getTypeHash(field.getDef(i));
@@ -698,6 +719,26 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
             return typeHash;
         }
 
+        if (field.isBlob())
+        {
+            FieldDef nonBlobField = new FieldDef(field);
+            nonBlobField.setIsBlob(false);
+
+            int nonBlobTypeHash = getJsonTypeDefinition(nonBlobField, typeDefinitionMap, typeDefinitions);
+            int nonBlobTypeIndex = typeDefinitionMap.get(nonBlobTypeHash);
+            String nonBlobTypeName = ESP_TYPE_NAME_PREFIX + (nonBlobTypeIndex + 1);
+
+            JSONObject typeDef = new JSONObject();
+            typeDef.put("fieldType", type_blob);
+            typeDef.put("length", 8);
+            typeDef.put("child", nonBlobTypeName);
+
+            int newTypeIndex = typeDefinitions.size();
+            typeDefinitions.add(typeDef);
+            typeDefinitionMap.put(typeHash, newTypeIndex);
+            return typeHash;
+        }
+
         JSONObject typeDef = new JSONObject();
         int typeID = getTypeID(field);
         switch (field.getFieldType())
@@ -753,25 +794,30 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
                     int childTypeHash = getJsonTypeDefinition(childField, typeDefinitionMap, typeDefinitions);
                     int childTypeIndex = typeDefinitionMap.get(childTypeHash);
                     String childTypeName = ESP_TYPE_NAME_PREFIX + (childTypeIndex + 1);
+
                     int childTypeID = getTypeID(childField);
+                    if (childField.isBlob())
+                    {
+                        childTypeID = type_blob;
+                    }
 
                     JSONObject childJson = new JSONObject();
-                    childJson.put("name", childField.getFieldName());
-                    childJson.put("type", childTypeName);
+                    childJson.put(NAME_KEY, childField.getFieldName());
+                    childJson.put(TYPE_KEY, childTypeName);
 
                     int flags = childTypeID | childField.getAdditionalFlags();
                     if (flags > 0)
                     {
-                        childJson.put("flags", flags);
+                        childJson.put(FLAGS_KEY, flags);
                     }
 
                     if (childField.getFieldType() == FieldType.DATASET)
                     {
-                        childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Row");
+                        childJson.put(XPATH_KEY, childField.getFieldName() + XPATH_DELIMITER + "Row");
                     }
                     else if (childField.getFieldType() == FieldType.SET)
                     {
-                        childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Item");
+                        childJson.put(XPATH_KEY, childField.getFieldName() + XPATH_DELIMITER + "Item");
                     }
 
                     fields.put(childJson);
@@ -954,6 +1000,14 @@ private static FieldDef parseJsonTypeDefinition(JSONObject jsonTypeDefinitions,
         int typeID = typeDef.getInt(FIELD_TYPE_KEY);
         long length = typeDef.getLong(LENGTH_KEY);
 
+        if (typeID == type_blob)
+        {
+            String blobType = typeDef.getString(CHILD_KEY);
+            FieldDef def = getOrParseJsonTypeDefintion(blobType, jsonTypeDefinitions, protoTypeDefs);
+            def.setIsBlob(true);
+            return def;
+        }
+
         FieldType fieldType = getFieldType(typeID);
         switch (fieldType)
         {
diff --git a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java
@@ -64,6 +64,22 @@ public class TestFieldDefinitions
             + "   \"name\": \"isactive\",\r\n   \"type\": \"ty15\",\r\n   \"flags\": 65536\r\n  },\r\n  {\r\n   \"name\": \"__internal_fpos__\",\r\n \"type\": \"ty16\",\r\n"
             + "   \"flags\": 65821\r\n  }\r\n ]\r\n}";
 
+    private static final String blobIndexDefinitionStr = "{\n \"ty6\": {  \"vinit\": 2, \"length\": 8, \"fieldType\": 285 },\n"
+            + "  \"ty5\": { \"length\": 8, \"fieldType\": 15, \"child\": \"ty4\" },\n"
+            + "  \"length\": 36,\n"
+            + "  \"ty2\": { \"length\": 0, \"fieldType\": 1028 },\n"
+            + "  \"fields\": [\n"
+            + "    { \"name\": \"str12\", \"flags\": 4, \"type\": \"ty1\" },\n"
+            + "    { \"name\": \"content_string\", \"flags\": 65551, \"type\": \"ty3\" },\n"
+            + "    { \"name\": \"content_data\", \"flags\": 65551, \"type\": \"ty5\" },\n"
+            + "    { \"name\": \"__internal_fpos__\", \"flags\": 65821, \"type\": \"ty6\" }\n"
+            + "  ],\n"
+            + "  \"ty1\": { \"length\": 12, \"fieldType\": 4 },\n"
+            + "  \"fieldType\": 13,\n"
+            + "  \"ty4\": { \"length\": 0, \"fieldType\": 1040 },\n"
+            + "  \"ty3\": { \"length\": 8, \"fieldType\": 15, \"child\": \"ty2\" }\n"
+            + "}\n";
+
     /**
      * Gets the complex record definition json.
      *
@@ -79,6 +95,11 @@ public static String getAllTypesIndexRecordDefinitionJson()
         return allTypesIndexRecordDefinitionStr;
     }
 
+    public static String getBlobIndexDefinitionJson()
+    {
+        return blobIndexDefinitionStr;
+    }
+
     /**
      * Gets the complex record definition.
      *
diff --git a/commons-hpcc/src/test/java/org/hpccsystems/commons/ecl/RecordDefinitionTests.java b/commons-hpcc/src/test/java/org/hpccsystems/commons/ecl/RecordDefinitionTests.java
@@ -39,7 +39,8 @@ public void setup()
     public void testJsonRecordParsing() throws Exception
     {
         String[] recordDefStrs = new String[] { TestFieldDefinitions.getComplexRecordDefinitionJson(),
-                                                 TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson() };
+                                                 TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson(),
+                                                 TestFieldDefinitions.getBlobIndexDefinitionJson() };
         for (String recordDefStr : recordDefStrs)
         {
             try
diff --git a/dfsclient/src/main/java/org/hpccsystems/dfs/client/HPCCFile.java b/dfsclient/src/main/java/org/hpccsystems/dfs/client/HPCCFile.java
@@ -250,6 +250,10 @@ private void updateProjectedRecordDef() throws Exception
                 field.setSourceType(HpccSrcType.LITTLE_ENDIAN);
             }
 
+            // if (field.isBlob())
+            // {
+            //     field.setIsBlob(false);
+            // }
         }
     }
 
diff --git a/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSIndexTest.java b/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSIndexTest.java
@@ -260,6 +260,27 @@ public void biasedIntTest() throws Exception
         }
     }
 
+    @Test
+    public void indexBlobTest() throws Exception
+    {
+        HPCCFile file = new HPCCFile("~test::index::blobs::key", connString , hpccUser, hpccPass);
+
+        DataPartition[] fileParts = file.getFileParts();
+
+        List<HPCCRecord> records = new ArrayList<HPCCRecord>();
+        FieldDef originalRD = file.getRecordDefinition();
+        for (int j = 0; j < fileParts.length; j++)
+        {
+            HPCCRecordBuilder recordBuilder = new HPCCRecordBuilder(file.getProjectedRecordDefinition());
+            HpccRemoteFileReader<HPCCRecord> fileReader = new HpccRemoteFileReader<HPCCRecord>(fileParts[j], originalRD, recordBuilder);
+            while (fileReader.hasNext())
+            {
+                records.add(fileReader.next());
+            }
+            fileReader.close();
+        }
+    }
+
     private String partitionListToString(List<DataPartition> partitions)
     {
         String matchedPartitionStr = "[ ";
diff --git a/dfsclient/src/test/resources/generate-datasets.ecl b/dfsclient/src/test/resources/generate-datasets.ecl

Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,7 @@ public class RecordDefinitionTranslator`
`32`	`32`	`private static final String NAME_KEY = "name";`
`33`	`33`	`private static final String TYPE_KEY = "type";`
`34`	`34`	`private static final String CHILD_KEY = "child";`
	`35`	`+ private static final String XPATH_KEY = "xpath";`
`35`	`36`	`private static final String FLAGS_KEY = "flags";`
`36`	`37`
`37`	`38`	`private static final String ESP_TYPE_NAME_PREFIX = "ty";`
`@@ -48,6 +49,7 @@ public class RecordDefinitionTranslator`
`48`	`49`	`final private static int type_keyedint = 10; // Convert to integer`
`49`	`50`	`final private static int type_record = 13;`
`50`	`51`	`final private static int type_varstring = 14;`
	`52`	`+ final private static int type_blob = 15;`
`51`	`53`	`final private static int type_data = 16;`
`52`	`54`	`final private static int type_table = 20;`
`53`	`55`	`final private static int type_set = 21;`
`@@ -264,28 +266,32 @@ public static String toECLRecord(FieldDef field) throws Exception`
`264`	`266`	`*/`
`265`	`267`	`private static String getEClTypeDefinition(FieldDef field, HashMap<String, String> recordDefinitionMap) throws Exception`
`266`	`268`	`{`
	`269`	`+ String type = "";`
`267`	`270`	`switch (field.getFieldType())`
`268`	`271`	`{`
`269`	`272`	`case SET:`
`270`	`273`	`{`
`271`		`- return "SET OF " + getEClTypeDefinition(field.getDef(0), recordDefinitionMap);`
	`274`	`+ type = "SET OF " + getEClTypeDefinition(field.getDef(0), recordDefinitionMap);`
	`275`	`+ break;`
`272`	`276`	`}`
`273`	`277`	`case DATASET:`
`274`	`278`	`{`
`275`		`- return "DATASET(" + getEClTypeDefinition(field.getDef(0), recordDefinitionMap) + ")";`
	`279`	`+ type = "DATASET(" + getEClTypeDefinition(field.getDef(0), recordDefinitionMap) + ")";`
	`280`	`+ break;`
`276`	`281`	`}`
`277`	`282`	`case BINARY:`
`278`	`283`	`{`
	`284`	`+ type = "DATA";`
`279`	`285`	`if (field.isFixed())`
`280`	`286`	`{`
`281`		`- return "DATA" + field.getDataLen();`
	`287`	`+ type += field.getDataLen();`
`282`	`288`	`}`
`283`		`-`
`284`		`- return "DATA";`
	`289`	`+ break;`
`285`	`290`	`}`
`286`	`291`	`case BOOLEAN:`
`287`	`292`	`{`
`288`		`- return "BOOLEAN";`
	`293`	`+ type = "BOOLEAN";`
	`294`	`+ break;`
`289`	`295`	`}`
`290`	`296`	`case INTEGER:`
`291`	`297`	`{`
`@@ -300,7 +306,8 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin`
`300`	`306`	`throw new Exception("Error: Unsupported integer size: " + field.getDataLen() + " must 1-8.");`
`301`	`307`	`}`
`302`	`308`
`303`		`- return root + field.getDataLen();`
	`309`	`+ type = root + field.getDataLen();`
	`310`	`+ break;`
`304`	`311`	`}`
`305`	`312`	`case FILEPOS:`
`306`	`313`	`{`
`@@ -314,7 +321,8 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin`
`314`	`321`	`throw new Exception("Error: Unsupported filepos size: " + field.getDataLen() + " must be 8.");`
`315`	`322`	`}`
`316`	`323`
`317`		`- return "UNSIGNED8";`
	`324`	`+ type = "UNSIGNED8";`
	`325`	`+ break;`
`318`	`326`	`}`
`319`	`327`	`case DECIMAL:`
`320`	`328`	`{`
`@@ -323,28 +331,33 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin`
`323`	`331`	`{`
`324`	`332`	`root = "U" + root;`
`325`	`333`	`}`
`326`		`- return root + field.getPrecision() + "_" + field.getScale();`
	`334`	`+ type = root + field.getPrecision() + "_" + field.getScale();`
	`335`	`+ break;`
`327`	`336`	`}`
`328`	`337`	`case REAL:`
`329`	`338`	`{`
`330`	`339`	`if (field.getDataLen() == 4)`
`331`	`340`	`{`
`332`		`- return "REAL4";`
	`341`	`+ type = "REAL4";`
`333`	`342`	`}`
`334`	`343`	`else if (field.getDataLen() == 8)`
`335`	`344`	`{`
`336`		`- return "REAL8";`
	`345`	`+ type = "REAL8";`
`337`	`346`	`}`
	`347`	`+ else`
	`348`	`+ {`
	`349`	`+ throw new Exception("Error: Unsupported real size: " + field.getDataLen() + " must 4 or 8.");`
	`350`	`+ }`
	`351`	`+ break;`
`338`	`352`
`339`		`- throw new Exception("Error: Unsupported real size: " + field.getDataLen() + " must 4 or 8.");`
`340`	`353`	`}`
`341`	`354`	`case CHAR:`
`342`	`355`	`{`
`343`		`- return "STRING1";`
	`356`	`+ type = "STRING1";`
	`357`	`+ break;`
`344`	`358`	`}`
`345`	`359`	`case STRING:`
`346`	`360`	`{`
`347`		`- String type = "";`
`348`	`361`	`HpccSrcType srcType = field.getSourceType();`
`349`	`362`	`if (srcType == HpccSrcType.SINGLE_BYTE_CHAR)`
`350`	`363`	`{`
`@@ -371,11 +384,10 @@ else if (srcType == HpccSrcType.QSTRING)`
`371`	`384`	`{`
`372`	`385`	`type += field.getDataLen();`
`373`	`386`	`}`
`374`		`- return type;`
	`387`	`+ break;`
`375`	`388`	`}`
`376`	`389`	`case VAR_STRING:`
`377`	`390`	`{`
`378`		`- String type = "";`
`379`	`391`	`HpccSrcType srcType = field.getSourceType();`
`380`	`392`	`if (srcType == HpccSrcType.SINGLE_BYTE_CHAR)`
`381`	`393`	`{`
`@@ -394,7 +406,7 @@ else if (srcType == HpccSrcType.UTF16LE \|\| srcType == HpccSrcType.UTF16BE)`
`394`	`406`	`{`
`395`	`407`	`type += field.getDataLen();`
`396`	`408`	`}`
`397`		`- return type;`
	`409`	`+ break;`
`398`	`410`	`}`
`399`	`411`	`case RECORD:`
`400`	`412`	`{`
`@@ -416,13 +428,21 @@ else if (srcType == HpccSrcType.UTF16LE \|\| srcType == HpccSrcType.UTF16BE)`
`416`	`428`	`String recordDefnName = "##" + hash + "##";`
`417`	`429`
`418`	`430`	`recordDefinitionMap.put(recordDefnName, definition);`
`419`		`- return recordDefnName;`
	`431`	`+ type = recordDefnName;`
	`432`	`+ break;`
`420`	`433`	`}`
`421`	`434`	`default:`
`422`	`435`	`{`
`423`	`436`	`throw new Exception("Unable to generate ECL unknown field type: " + field.getFieldType().description());`
`424`	`437`	`}`
`425`	`438`	`}`
	`439`	`+`
	`440`	`+ if (field.isBlob())`
	`441`	`+ {`
	`442`	`+ type += " {blob}";`
	`443`	`+ }`
	`444`	`+`
	`445`	`+ return type;`
`426`	`446`	`}`
`427`	`447`
`428`	`448`	`/**`
`@@ -648,7 +668,7 @@ else if (srcType == HpccSrcType.UTF16LE \|\| srcType == HpccSrcType.UTF16BE)`
`648`	`668`	`*/`
`649`	`669`	`private static int getTypeHash(FieldDef field) throws Exception`
`650`	`670`	`{`
`651`		`- int numHashComponents = 4 + field.getNumDefs();`
	`671`	`+ int numHashComponents = 5 + field.getNumDefs();`
`652`	`672`	`if (field.getFieldType() == FieldType.DECIMAL)`
`653`	`673`	`{`
`654`	`674`	`numHashComponents += 2;`
`@@ -659,8 +679,9 @@ private static int getTypeHash(FieldDef field) throws Exception`
`659`	`679`	`hashComponents[1] = field.getDataLen();`
`660`	`680`	`hashComponents[2] = field.getSourceType().ordinal();`
`661`	`681`	`hashComponents[3] = field.getAdditionalFlags();`
	`682`	`+ hashComponents[4] = field.isBlob() ? 1 : 0;`
`662`	`683`
`663`		`- int hashCompIndex = 4;`
	`684`	`+ int hashCompIndex = 5;`
`664`	`685`	`for (int i = 0; i < field.getNumDefs(); i++, hashCompIndex++)`
`665`	`686`	`{`
`666`	`687`	`hashComponents[hashCompIndex] = getTypeHash(field.getDef(i));`
`@@ -698,6 +719,26 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege`
`698`	`719`	`return typeHash;`
`699`	`720`	`}`
`700`	`721`
	`722`	`+ if (field.isBlob())`
	`723`	`+ {`
	`724`	`+ FieldDef nonBlobField = new FieldDef(field);`
	`725`	`+ nonBlobField.setIsBlob(false);`
	`726`	`+`
	`727`	`+ int nonBlobTypeHash = getJsonTypeDefinition(nonBlobField, typeDefinitionMap, typeDefinitions);`
	`728`	`+ int nonBlobTypeIndex = typeDefinitionMap.get(nonBlobTypeHash);`
	`729`	`+ String nonBlobTypeName = ESP_TYPE_NAME_PREFIX + (nonBlobTypeIndex + 1);`
	`730`	`+`
	`731`	`+ JSONObject typeDef = new JSONObject();`
	`732`	`+ typeDef.put("fieldType", type_blob);`
	`733`	`+ typeDef.put("length", 8);`
	`734`	`+ typeDef.put("child", nonBlobTypeName);`
	`735`	`+`
	`736`	`+ int newTypeIndex = typeDefinitions.size();`
	`737`	`+ typeDefinitions.add(typeDef);`
	`738`	`+ typeDefinitionMap.put(typeHash, newTypeIndex);`
	`739`	`+ return typeHash;`
	`740`	`+ }`
	`741`	`+`
`701`	`742`	`JSONObject typeDef = new JSONObject();`
`702`	`743`	`int typeID = getTypeID(field);`
`703`	`744`	`switch (field.getFieldType())`
`@@ -753,25 +794,30 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege`
`753`	`794`	`int childTypeHash = getJsonTypeDefinition(childField, typeDefinitionMap, typeDefinitions);`
`754`	`795`	`int childTypeIndex = typeDefinitionMap.get(childTypeHash);`
`755`	`796`	`String childTypeName = ESP_TYPE_NAME_PREFIX + (childTypeIndex + 1);`
	`797`	`+`
`756`	`798`	`int childTypeID = getTypeID(childField);`
	`799`	`+ if (childField.isBlob())`
	`800`	`+ {`
	`801`	`+ childTypeID = type_blob;`
	`802`	`+ }`
`757`	`803`
`758`	`804`	`JSONObject childJson = new JSONObject();`
`759`		`- childJson.put("name", childField.getFieldName());`
`760`		`- childJson.put("type", childTypeName);`
	`805`	`+ childJson.put(NAME_KEY, childField.getFieldName());`
	`806`	`+ childJson.put(TYPE_KEY, childTypeName);`
`761`	`807`
`762`	`808`	`int flags = childTypeID \| childField.getAdditionalFlags();`
`763`	`809`	`if (flags > 0)`
`764`	`810`	`{`
`765`		`- childJson.put("flags", flags);`
	`811`	`+ childJson.put(FLAGS_KEY, flags);`
`766`	`812`	`}`
`767`	`813`
`768`	`814`	`if (childField.getFieldType() == FieldType.DATASET)`
`769`	`815`	`{`
`770`		`- childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Row");`
	`816`	`+ childJson.put(XPATH_KEY, childField.getFieldName() + XPATH_DELIMITER + "Row");`
`771`	`817`	`}`
`772`	`818`	`else if (childField.getFieldType() == FieldType.SET)`
`773`	`819`	`{`
`774`		`- childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Item");`
	`820`	`+ childJson.put(XPATH_KEY, childField.getFieldName() + XPATH_DELIMITER + "Item");`
`775`	`821`	`}`
`776`	`822`
`777`	`823`	`fields.put(childJson);`
`@@ -954,6 +1000,14 @@ private static FieldDef parseJsonTypeDefinition(JSONObject jsonTypeDefinitions,`
`954`	`1000`	`int typeID = typeDef.getInt(FIELD_TYPE_KEY);`
`955`	`1001`	`long length = typeDef.getLong(LENGTH_KEY);`
`956`	`1002`
	`1003`	`+ if (typeID == type_blob)`
	`1004`	`+ {`
	`1005`	`+ String blobType = typeDef.getString(CHILD_KEY);`
	`1006`	`+ FieldDef def = getOrParseJsonTypeDefintion(blobType, jsonTypeDefinitions, protoTypeDefs);`
	`1007`	`+ def.setIsBlob(true);`
	`1008`	`+ return def;`
	`1009`	`+ }`
	`1010`	`+`
`957`	`1011`	`FieldType fieldType = getFieldType(typeID);`
`958`	`1012`	`switch (fieldType)`
`959`	`1013`	`{`
Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,8 @@ public void setup()`
`39`	`39`	`public void testJsonRecordParsing() throws Exception`
`40`	`40`	`{`
`41`	`41`	`String[] recordDefStrs = new String[] { TestFieldDefinitions.getComplexRecordDefinitionJson(),`
`42`		`- TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson() };`
	`42`	`+ TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson(),`
	`43`	`+ TestFieldDefinitions.getBlobIndexDefinitionJson() };`
`43`	`44`	`for (String recordDefStr : recordDefStrs)`
`44`	`45`	`{`
`45`	`46`	`try`
Original file line number	Diff line number	Diff line change
`@@ -250,6 +250,10 @@ private void updateProjectedRecordDef() throws Exception`
`250`	`250`	`field.setSourceType(HpccSrcType.LITTLE_ENDIAN);`
`251`	`251`	`}`
`252`	`252`
	`253`	`+ // if (field.isBlob())`
	`254`	`+ // {`
	`255`	`+ // field.setIsBlob(false);`
	`256`	`+ // }`
`253`	`257`	`}`
`254`	`258`	`}`
`255`	`259`