improve json

src/decoders.js

@@ -20,7 +20,7 @@ export const wireLabels = {
 export const wireMap = {
   0: ['int', 'bool', 'raw'],
   1: ['int', 'uint', 'bytes', 'float', 'raw'],
-  2: ['string', 'bytes', 'sub', 'packedvarint', 'packedint32', 'packedint64', 'raw'],
+  2: ['bytes', 'string', 'sub', 'packedvarint', 'packedint32', 'packedint64', 'raw'],
   5: ['int', 'uint', 'bytes', 'float', 'raw']
 }
 

src/index.js

@@ -55,15 +55,6 @@ export class RawProto {
     this.choices = choices
   }
 
-  query(path, prefix = '') {
-    if (!this.tree) {
-      this.offset = 0
-      this.tree = this.readMessage()
-      this.offset = 0
-    }
-    return query(this.tree, path, this.choices, prefix)
-  }
-
   // read a VARINT from buffer, at offset
   readVarInt() {
     let result = 0
@@ -102,15 +93,23 @@ export class RawProto {
     return this.buffer.slice(offsetStart, this.offset)
   }
 
-  handleField(type, index) {
-    // choose first renderType as default
-    // TODO: handle choices
-    let renderType = 'raw'
-    if (wireMap[type]) {
-      renderType = wireMap[type][0]
+  handleField(type, index, prefix = '') {
+    const path = prefix ? `${prefix}.${index}` : index.toString()
+
+    // choose first renderType as default, if chjoice not set
+    let renderType = this.choices[path]
+    if (!renderType) {
+      if (wireMap[type]) {
+        renderType = wireMap[type][0]
+      }
+    }
+
+    const newrec = { path, type, index, pos: [this.offset], renderType }
+
+    if (type === wireTypes.SGROUP || type === wireTypes.EGROUP || type > wireTypes.I32) {
+      delete newrec.renderType
     }
 
-    const newrec = { type, index, pos: [this.offset], renderType }
     switch (type) {
       case wireTypes.VARINT:
         newrec.value = this.readVarInt()
@@ -122,14 +121,11 @@ export class RawProto {
         return newrec
       case wireTypes.LEN:
         newrec.value = this.readBuffer(this.readVarInt())
+        newrec.value = decoders.getValue(newrec, renderType)
         newrec.pos.push(this.offset)
-        // this checks if sub-message is possible
-        // try {
-        //   newrec.sub = new RawProto(newrec.value).readMessage()
-        //   newrec.renderType = 'sub'
-        // } catch (e) {}
         return newrec
       case wireTypes.SGROUP:
+        // just get bytes
         newrec.value = this.readGroup(index)
         newrec.pos.push(this.offset)
         return newrec
@@ -146,19 +142,28 @@ export class RawProto {
   }
 
   // read 1 level of LEN message field
-  readMessage() {
+  readMessage(prefix = '') {
     // TODO: use this.choices to insert renderType
     const out = []
     while (this.offset < this.buffer.length) {
       const indexType = parseInt(this.readVarInt())
-      const r = this.handleField(indexType & 0b111, indexType >> 3)
+      const r = this.handleField(indexType & 0b111, indexType >> 3, prefix)
       if (r) {
         out.push(r)
       }
     }
     return out
   }
 
+  // These are wrapopers around other utils that could be used seperately
+
+  query(path, prefix = '') {
+    this.offset = 0
+    this.tree ||= this.readMessage(prefix)
+    this.offset = 0
+    return query(this.tree, path, this.choices, prefix)
+  }
+
   walk(callback, prefix = '') {
     return walk(this, callback, prefix)
   }
@@ -184,21 +189,32 @@ export const removePathPrefix = (choices = {}, path = '') =>
 
 // walk over a tree recursivley calling callback on each item, each field is outputted as an array, eah message is an object
 export function walk(reader, callback, prefix = '') {
-  reader.tree ||= reader.readMessage()
+  reader.offset = 0
+  reader.tree ||= reader.readMessage(prefix)
+  reader.offset = 0
   const out = {}
   for (const field of reader.tree) {
     out[field.index] ||= []
-    const f = prefix ? `${prefix}.${field.index}` : field.index
-    out[field.index].push(callback(field, f, reader, callback))
+    const path = prefix ? `${prefix}.${field.index}` : field.index
+    out[field.index].push(callback(field, path, reader, callback))
   }
   return out
 }
 
 // generic walker that will apply default transforms to every field
 export function walkerJS(field, path, reader, callback) {
   if (field.type === wireTypes.LEN) {
-    // TODO: pass choices form original reader
-    return walk(new RawProto(field.value, removePathPrefix(reader.choices, path)), callback, path)
+    try {
+      return walk(new RawProto(field.value, removePathPrefix(reader.choices, path)), callback, path)
+    } catch (e) {
+      // check if it is likely a string
+      for (const b of field.value) {
+        if (b < 32) {
+          return decoders.getValue(field, 'bytes')
+        }
+      }
+      return decoders.getValue(field, 'string')
+    }
   }
   return decoders.getValue(field, field.renderType)
 }

test/json.test.js

@@ -20,31 +20,29 @@ test('Submessages', () => {
   expect(j).toEqual({ 3: [{ 1: [150] }] })
 })
 
-// TODO: these are failing
-
-test.skip('Length-Delimited Records', () => {
+test('Length-Delimited Records', () => {
   /*
     2: {"testing"}
   */
-  const j = new RawProto([0x12, 0x07, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67], { 2: 'string' }).toJS()
+  const j = new RawProto([0x12, 0x07, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67]).toJS()
   expect(j).toEqual({ 2: ['testing'] })
 })
 
-test.skip('Repeated Elements', () => {
+test('Repeated Elements', () => {
   /*
     4: {"hello"}
     5: 1
     5: 2
     5: 3
   */
-  const j = new RawProto([0x22, 0x05, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x28, 0x01, 0x28, 0x02, 0x28, 0x03], { 4: 'string' }).toJS()
+  const j = new RawProto([0x22, 0x05, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x28, 0x01, 0x28, 0x02, 0x28, 0x03]).toJS()
   expect(j).toEqual({ 4: ['hello'], 5: [1, 2, 3] })
 })
 
-test.skip('Packed Repeated Fields', () => {
+test('Packed Repeated Fields', () => {
   /*
     6: {3 270 86942}
   */
   const j = new RawProto([0x32, 0x06, 0x03, 0x8e, 0x02, 0x9e, 0xa7, 0x05], { 6: 'packedvarint' }).toJS()
-  expect(j).toEqual({ 6: [3, 270, 86942] })
+  expect(j).toEqual({ 6: [[3, 270, 86942]] })
 })

test/query.test.js

@@ -55,7 +55,7 @@ test('Length-Delimited Records', () => {
   expect(query(tree, '2:string')).toEqual(['testing'])
 })
 
-test('Submessages', () => {
+test('Traverse Submessages', () => {
   /*
     3: {1: 150}
   */