ENH: Preserve key order when passing list of dicts to DataFrame on py 3.6+

pandas/core/frame.py

@@ -439,9 +439,28 @@ def __init__(self, data=None, index=None, columns=None, dtype=None, copy=False):
                 data = list(data)
             if len(data) > 0:
                 if is_list_like(data[0]) and getattr(data[0], "ndim", 1) == 1:
-                    if is_named_tuple(data[0]) and columns is None:
+                    infer_columns_names = columns is None
+                    if is_named_tuple(data[0]) and infer_columns_names:
                         columns = data[0]._fields
-                    arrays, columns = to_arrays(data, columns, dtype=dtype)
+                    arrays, arr_names = to_arrays(data, columns, dtype=dtype)
+                    arr_names = ensure_index(arr_names)
+
+                    columns = arr_names
+                    # GH#10056
+                    if (
+                        PY36
+                        and is_dict_like(data[0])
+                        and infer_columns_names
+                        and (type(columns) is Index)
+                    ):
+                        _columns = list(columns)
+                        known_columns = set(data[0])
+                        extra_columns = [_ for _ in _columns if _ not in known_columns]
+                        if set(_columns[: len(data[0])]) == known_columns:
+                            _columns[: len(known_columns)] = list(data[0])
+                            _columns[len(known_columns) :] = extra_columns
+                            columns = _columns
+
                     columns = ensure_index(columns)
 
                     # set the index
@@ -453,7 +472,7 @@ def __init__(self, data=None, index=None, columns=None, dtype=None, copy=False):
                         else:
                             index = ibase.default_index(len(data))
 
-                    mgr = arrays_to_mgr(arrays, columns, index, columns, dtype=dtype)
+                    mgr = arrays_to_mgr(arrays, arr_names, index, columns, dtype=dtype)
                 else:
                     mgr = init_ndarray(data, index, columns, dtype=dtype, copy=copy)
             else:

pandas/tests/frame/test_constructors.py

@@ -1119,7 +1119,7 @@ def test_constructor_generator(self):
         expected = DataFrame({0: range(10), 1: "a"})
         tm.assert_frame_equal(result, expected, check_dtype=False)
 
-    def test_constructor_list_of_dicts(self):
+    def test_constructor_list_of_odicts(self):
         data = [
             OrderedDict([["a", 1.5], ["b", 3], ["c", 4], ["d", 6]]),
             OrderedDict([["a", 1.5], ["b", 3], ["d", 6]]),
@@ -1340,6 +1340,31 @@ def test_constructor_list_of_namedtuples(self):
         result = DataFrame(tuples, columns=["y", "z"])
         tm.assert_frame_equal(result, expected)
 
+    @pytest.mark.skipif(not PY36, reason="Guaranteed dict order is Python>=3.7")
+    def test_constructor_list_of_dict_order(self):
+        # GH10056
+        data = [
+            {"First": 1, "Second": 4, "Third": 7, "Fourth": 10},
+            {"Second": 5, "First": 2, "Fourth": 11, "Third": 8},
+            {"Second": 6, "First": 3, "Fourth": 12, "Third": 9, "YYY": 14, "XXX": 13},
+        ]
+        expected = DataFrame(
+            {
+                "First": [1, 2, 3],
+                "Second": [4, 5, 6],
+                "Third": [7, 8, 9],
+                "Fourth": [10, 11, 12],
+                "XXX": [None, None, 13],
+                "YYY": [None, None, 14],
+            }
+        )
+        result = DataFrame(data)
+        assert set(result.columns) == set(expected.columns)
+        # order of first 4 columns dictated by data[0]
+        tm.assert_frame_equal(result.iloc[:, :4], expected.iloc[:, :4])
+        # the order of the remaining two is arbitrary, but assert the data matches
+        tm.assert_frame_equal(result[["XXX", "YYY"]], expected[["XXX", "YYY"]])
+
     def test_constructor_orient(self, float_string_frame):
         data_dict = float_string_frame.T._series
         recons = DataFrame.from_dict(data_dict, orient="index")

pandas/tests/io/json/test_normalize.py

@@ -3,6 +3,8 @@
 import numpy as np
 import pytest
 
+from pandas.compat import PY36
+
 from pandas import DataFrame, Index
 import pandas.util.testing as tm
 
@@ -351,9 +353,9 @@ def test_non_ascii_key(self):
         ).decode("utf8")
 
         testdata = {
+            b"\xc3\x9cnic\xc3\xb8de".decode("utf8"): [0, 1],
             "sub.A": [1, 3],
             "sub.B": [2, 4],
-            b"\xc3\x9cnic\xc3\xb8de".decode("utf8"): [0, 1],
         }
         expected = DataFrame(testdata)
 
@@ -365,16 +367,16 @@ def test_missing_field(self, author_missing_data):
         result = json_normalize(author_missing_data)
         ex_data = [
             {
-                "info": np.nan,
                 "author_name.first": np.nan,
                 "author_name.last_name": np.nan,
+                "info": np.nan,
                 "info.created_at": np.nan,
                 "info.last_updated": np.nan,
             },
             {
-                "info": None,
                 "author_name.first": "Jane",
                 "author_name.last_name": "Doe",
+                "info": None,
                 "info.created_at": "11/08/1993",
                 "info.last_updated": "26/05/2012",
             },
@@ -508,11 +510,17 @@ def test_missing_meta(self, missing_metadata):
             data=missing_metadata, record_path="addresses", meta="name", errors="ignore"
         )
         ex_data = [
-            ["Massillon", 9562, "OH", "Morris St.", 44646, "Alice"],
-            ["Elizabethton", 8449, "TN", "Spring St.", 37643, np.nan],
+            [9562, "Morris St.", "Massillon", "OH", 44646, "Alice"],
+            [8449, "Spring St.", "Elizabethton", "TN", 37643, np.nan],
         ]
-        columns = ["city", "number", "state", "street", "zip", "name"]
+        columns = ["number", "street", "city", "state", "zip", "name"]
         expected = DataFrame(ex_data, columns=columns)
+        if not PY36:
+            # json_normalize order is not guaranteed, so columns
+            # depends on implementation. Opt to test on PY36/37
+            # and force column order on PY35.
+            expected = expected[columns]
+            result = result[columns]
         tm.assert_frame_equal(result, expected)
 
     def test_donot_drop_nonevalues(self):