Compatibility

- setting BIT_PACKED encoding confuses spark so it's now set to RLE
- custom metadata supported on read and write

Author: Ivan Gavryliuk

src/Parquet.Test/MapsTest.cs

@@ -12,21 +12,21 @@ public class MapsTest
       public void Simple_first_level_map_int_to_string()
       {
          var ds = new DataSet(
-            new SchemaElement<int>("id"),
-            new SchemaElement<IDictionary<int, string>>("names"));
+            new SchemaElement<IDictionary<int, string>>("names"),
+            new SchemaElement<int>("id"));
 
-         ds.Add(1, new Dictionary<int, string>
+         ds.Add(new Dictionary<int, string>
          {
             [1] = "one",
             [2] = "two",
             [3] = "three"
-         });
+         }, 1);
 
-         //ParquetWriter.WriteFile(ds, "c:\\tmp\\map.parquet");
+         //ParquetWriter.WriteFile(ds, "c:\\tmp\\pmap2.parquet");
 
          DataSet ds1 = DataSetGenerator.WriteRead(ds);
 
-         Assert.Equal("{1;[1=>one;2=>two;3=>three]}", ds1[0].ToString());
+         Assert.Equal("{[1=>one;2=>two;3=>three];1}", ds1[0].ToString());
       }
    }
 }

src/Parquet.Test/ParquetReaderTest.cs

@@ -283,6 +283,9 @@ public void Read_simple_map()
          Assert.Equal(3, ms.Extra[1].MaxDefinitionLevel);
 
          Assert.Equal("{1;[1=>one;2=>two;3=>three]}", ds[0].ToString());
+
+         //DataSet ds2 = DataSetGenerator.WriteRead(ds);
+         //ParquetWriter.WriteFile(ds, "c:\\tmp\\pmaps.parquet", CompressionMethod.None);
       }
 
       [Fact]

src/Parquet/Data/DataSetMetadata.cs

@@ -1,4 +1,6 @@
-namespace Parquet.Data
+using System.Collections.Generic;
+
+namespace Parquet.Data
 {
    /// <summary>
    /// Public metadata
@@ -9,5 +11,10 @@ public class DataSetMetadata
       /// Gets the creator tag.
       /// </summary>
       public string CreatedBy { get; internal set; }
+
+      /// <summary>
+      /// Custom metadata properties
+      /// </summary>
+      public Dictionary<string, string> Custom { get; private set; } = new Dictionary<string, string>();
    }
 }

src/Parquet/File/FileMetadataBuilder.cs

@@ -53,14 +53,17 @@ public FileMetadataBuilder()
       public void AddSchema(DataSet ds)
       {
          ds.Metadata.CreatedBy = CreatedBy;
+
          _meta.Schema = new List<TSchemaElement> { new TSchemaElement("schema") { Num_children = ds.Schema.Elements.Count } };
+         _meta.Key_value_metadata = ds.Metadata.Custom.Select(kv => new Thrift.KeyValue(kv.Key) { Value = kv.Value }).ToList();
 
          foreach(SchemaElement se in ds.Schema.Elements)
          {
             AddSchema(_meta.Schema, se);
          }
 
          _meta.Num_rows = ds.Count;
+         
       }
 
       private static void AddSchema(List<TSchemaElement> container, SchemaElement se)
@@ -210,7 +213,7 @@ public Thrift.PageHeader CreateDataPage(int valueCount)
          {
             Encoding = Thrift.Encoding.PLAIN,
             Definition_level_encoding = Thrift.Encoding.RLE,
-            Repetition_level_encoding = Thrift.Encoding.BIT_PACKED,
+            Repetition_level_encoding = Thrift.Encoding.RLE,
             Num_values = valueCount
          };
 

src/Parquet/File/FileMetadataParser.cs

@@ -79,6 +79,19 @@ void Build(SchemaElement node, ref int i, int count, bool isRoot)
          return new Schema(root.Children);
       }
 
+      public void AddMeta(DataSet ds)
+      {
+         ds.Metadata.Custom.Clear();
+
+         if (_fileMeta.Key_value_metadata != null && _fileMeta.Key_value_metadata.Count > 0)
+         {
+            foreach(Thrift.KeyValue tkv in _fileMeta.Key_value_metadata)
+            {
+               ds.Metadata.Custom[tkv.Key] = tkv.Value;
+            }
+         }
+      }
+
       private SchemaElement BuildListSchema(ref Thrift.SchemaElement tse, ref int i, bool isRoot, SchemaElement node, ParquetOptions formatOptions)
       {
          Thrift.SchemaElement tseTop = tse;

src/Parquet/ParquetReader.cs

@@ -148,7 +148,10 @@ public DataSet Read()
             pos += rg.Num_rows;
          }
 
-         return new DataSet(schema, pathToValues, _meta.Num_rows, _meta.Created_by) { Thrift = _meta };
+         var ds = new DataSet(schema, pathToValues, _meta.Num_rows, _meta.Created_by);
+         metaParser.AddMeta(ds);
+         ds.Thrift = _meta;
+         return ds;
       }
 
       /// <summary>