Initial code for importing cred graphs (#10)

Dandelion Mané · web-flow · commit 02317bff5e6b · 2019-03-20T11:37:51.000-07:00
This commit contains inaugural infra code, which imports a SourceCred
graph and adds type annotations. The code itself is quite simple, with a
bit of logic for pulling out the most specific declared node/edge type and
adding it as a property in the networkx graph.

The logic is lightly tested using the sample graphs. We verify that each
graph loads, and for the sourcecred/sourcecred graph, we verify that we
have reasonable counts by node and edge type.

Test plan:

Run the included unit tests via
`python infra/import_graph_test.py`
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
diff --git a/infra/import_graph.py b/infra/import_graph.py
@@ -0,0 +1,130 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import networkx as nx
+import collections
+
+AddressType = collections.namedtuple("AddressType", ("prefix", "type"))
+
+
+def _type_prefix_match(address_types, address):
+    """For a given address, find the type matching the address.
+
+    Takes an object containing an array of {prefix, type} pairs, and
+    an address. Returns the first type whose corresponding prefix
+    was a prefix of the given address.
+    """
+    for address_type in address_types:
+        prefix = address_type.prefix
+        if address[: len(prefix)] == prefix:
+            return address_type.type
+    raise ValueError("No matching prefix for {}".format(address))
+
+
+def node_type(address):
+    """Return a string that identifies the "type" of a SourceCred node.
+
+    For any anticipated SourceCred node (i.e., it was supplied by one of the two
+    standard SourceCred plugins, i.e., sourcecred/git and sourcecred/github),
+    this method returns a string which identifies the most specific declared
+    node_type that matches the given node.
+
+    The SourceCred type system is still pretty ad-hoc,
+    (see: https://github.com/sourcecred/sourcecred/issues/710), so this system is
+    likely to change in the future.
+    """
+    NODE_PREFIX_TO_TYPE = [
+        AddressType(prefix=["sourcecred", "github", "REPO"], type="github/repo"),
+        AddressType(
+            prefix=["sourcecred", "github", "USERLIKE", "USER"], type="github/user"
+        ),
+        AddressType(
+            prefix=["sourcecred", "github", "USERLIKE", "BOT"], type="github/bot"
+        ),
+        AddressType(prefix=["sourcecred", "github", "PULL"], type="github/pull"),
+        AddressType(prefix=["sourcecred", "github", "ISSUE"], type="github/issue"),
+        AddressType(prefix=["sourcecred", "github", "REVIEW"], type="github/review"),
+        AddressType(prefix=["sourcecred", "github", "COMMENT"], type="github/comment"),
+        AddressType(prefix=["sourcecred", "git", "COMMIT"], type="git/commit"),
+    ]
+    return _type_prefix_match(NODE_PREFIX_TO_TYPE, address)
+
+
+def edge_type(address):
+    """Return a string that identifies the "type" of a SourceCred edge.
+
+    For any anticipated SourceCred edge (i.e., it was supplied by one of the two
+    standard SourceCred plugins, i.e., sourcecred/git and sourcecred/github),
+    this method returns a string which identifies the most specific declared
+    edge_type that matches the given node.
+
+    The SourceCred type system is still pretty ad-hoc,
+    (see: https://github.com/sourcecred/sourcecred/issues/710), so this system is
+    likely to change in the future.
+    """
+    EDGE_PREFIX_TO_TYPE = [
+        AddressType(
+            prefix=["sourcecred", "github", "HAS_PARENT"], type="github/hasParent"
+        ),
+        AddressType(
+            prefix=["sourcecred", "github", "REFERENCES"], type="github/references"
+        ),
+        AddressType(
+            prefix=["sourcecred", "github", "MENTIONS_AUTHOR"],
+            type="github/mentionsAuthor",
+        ),
+        AddressType(prefix=["sourcecred", "github", "AUTHORS"], type="github/authors"),
+        AddressType(prefix=["sourcecred", "github", "PULL"], type="github/pull"),
+        AddressType(prefix=["sourcecred", "github", "ISSUE"], type="github/issue"),
+        AddressType(prefix=["sourcecred", "github", "REVIEW"], type="github/review"),
+        AddressType(prefix=["sourcecred", "github", "COMMENT"], type="github/comment"),
+        AddressType(
+            prefix=["sourcecred", "github", "MERGED_AS"], type="github/mergedAs"
+        ),
+        AddressType(
+            prefix=["sourcecred", "github", "REACTS", "HOORAY"],
+            type="github/reactsHooray",
+        ),
+        AddressType(
+            prefix=["sourcecred", "github", "REACTS", "THUMBS_UP"],
+            type="github/reactsThumbsUp",
+        ),
+        AddressType(
+            prefix=["sourcecred", "github", "REACTS", "HEART"],
+            type="github/reactsHeart",
+        ),
+        AddressType(
+            prefix=["sourcecred", "github", "REACTS", "ROCKET"],
+            type="github/reactsRocket",
+        ),
+        AddressType(prefix=["sourcecred", "git", "HAS_PARENT"], type="git/hasParent"),
+    ]
+    return _type_prefix_match(EDGE_PREFIX_TO_TYPE, address)
+
+
+def json_to_graph(json):
+    """Convert a serialized SourceCred graph to a MultiDiGraph.
+
+    Takes in a Python dict representing a SourceCred graph json.
+    Returns a networkx MultiDiGraph, with node and edge type identifiers
+    added as an additional property.
+    """
+    [compat, data] = json
+    assert compat["type"] == "sourcecred/graph", compat
+    assert compat["version"] == "0.4.0", compat
+
+    def nodePropertyDict(address):
+        return {"address": tuple(address), "type": node_type(address)}
+
+    def edgePropertyDict(address):
+        return {"address": tuple(address), "type": edge_type(address)}
+
+    nodes = data["nodes"]
+    edges = data["edges"]
+    g = nx.MultiDiGraph()
+    for (i, n) in enumerate(nodes):
+        g.add_node(i, **nodePropertyDict(n))
+    for e in edges:
+        g.add_edge(e["srcIndex"], e["dstIndex"], **edgePropertyDict(e["address"]))
+    return g
diff --git a/infra/import_graph_test.py b/infra/import_graph_test.py
@@ -0,0 +1,131 @@
+import unittest
+import json
+import os
+from collections import Counter
+
+import import_graph
+
+
+class TestTypeMatching(unittest.TestCase):
+    def test_node_type_matching(self):
+        # Testing the basic functionality. Cases don't need to be exhaustive;
+        # missing types will be caught during the whole-graph import tests.
+        example_issue = [
+            "sourcecred",
+            "github",
+            "ISSUE",
+            "sourcecred",
+            "sourcecred",
+            "34",
+        ]
+        example_repo = ["sourcecred", "github", "REPO", "sourcecred", "sourcecred"]
+        self.assertEqual(import_graph.node_type(example_issue), "github/issue")
+        self.assertEqual(import_graph.node_type(example_repo), "github/repo")
+        with (self.assertRaises(ValueError)):
+            import_graph.node_type(["non", "existent", "node"])
+
+    def test_edge_type_matching(self):
+        # Testing the basic functionality. Cases don't need to be exhaustive;
+        # missing types will be caught during the whole-graph import tests.
+        example_has_parent = [
+            "sourcecred",
+            "github",
+            "HAS_PARENT",
+            "6",
+            "sourcecred",
+            "github",
+            "ISSUE",
+            "sourcecred",
+            "pm",
+            "1",
+        ]
+
+        example_authors = [
+            "sourcecred",
+            "github",
+            "AUTHORS",
+            "5",
+            "sourcecred",
+            "github",
+            "USERLIKE",
+            "USER",
+            "BrianLitwin",
+            "4",
+            "sourcecred",
+            "git",
+            "COMMIT",
+            "0cae9fa77c1d7d8b8fe3fe2d316a6782757862e4",
+        ]
+        self.assertEqual(import_graph.edge_type(example_has_parent), "github/hasParent")
+        self.assertEqual(import_graph.edge_type(example_authors), "github/authors")
+        with (self.assertRaises(ValueError)):
+            import_graph.edge_type(["non", "existent", "edge"])
+
+
+def sample_graphs_directory():
+    wd = os.path.abspath(os.path.dirname(__file__))
+    return os.path.join(wd, os.pardir, "sample-graphs")
+
+
+class TestImportGraph(unittest.TestCase):
+    def test_for_sourcecred_sourcecred(self):
+        sourcecred_sourcecred = os.path.join(
+            sample_graphs_directory(), "sourcecred_sourcecred.json"
+        )
+
+        with open(sourcecred_sourcecred, "r") as f:
+            sourcecred_graph_data = json.load(f)
+        graph = import_graph.json_to_graph(sourcecred_graph_data)
+
+        # Some sanity checks on the loaded graph, based on node type counts
+        # We don't expect exact counts because the expectation is that these graphs are
+        # regularly re-generated.
+        # Under normal circumstances the counts of these entities only increases,
+        # so this test should be reasonably robust.
+        node_count = Counter()
+        for n in graph.nodes(data=True):
+            node_count[n[1]["type"]] += 1
+
+        self.assertEqual(node_count["github/repo"], 1)
+        self.assertGreater(node_count["github/user"], 10)
+        self.assertGreater(node_count["github/issue"], 100)
+        self.assertGreater(node_count["github/pull"], 800)
+        self.assertGreater(node_count["github/review"], 200)
+        self.assertGreater(node_count["github/comment"], 500)
+        self.assertGreater(node_count["git/commit"], 800)
+
+        # Some sanity checks on the loaded graph, based on edge type counts.
+        # Same reasoning as the tests above.
+        edge_count = Counter()
+        for e in graph.edges(data=True):
+            edge_count[e[2]["type"]] += 1
+
+        self.assertGreater(edge_count["github/authors"], 1000)
+        self.assertGreater(edge_count["github/references"], 100)
+        self.assertGreater(edge_count["github/hasParent"], 1000)
+        self.assertGreater(edge_count["github/mergedAs"], 800)
+        self.assertGreater(edge_count["github/reactsThumbsUp"], 20)
+        self.assertGreater(edge_count["github/reactsHooray"], 5)
+        self.assertGreater(edge_count["github/reactsHeart"], 10)
+        self.assertGreater(edge_count["github/reactsRocket"], 1)
+        self.assertGreater(edge_count["git/hasParent"], 800)
+
+    def test_all_graphs_load(self):
+        files = os.listdir(sample_graphs_directory())
+        graph_files = [
+            os.path.join(sample_graphs_directory(), f)
+            for f in files
+            if f.endswith(".json")
+        ]
+        self.assertGreater(len(graph_files), 3)
+        for g in graph_files:
+            if g.endswith("sourcecred_sourcecred.json"):
+                continue  # This one is loaded in a separate test case.
+            with open(g, "r") as f:
+                data = json.load(f)
+            # Just verify it doesn't throw an error.
+            import_graph.json_to_graph(data)
+
+
+if __name__ == "__main__":
+    unittest.main()

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+# Byte-compiled / optimized / DLL files`
	`2`	`+__pycache__/`
	`3`	`+*.py[cod]`