schema/context: restore some backlinks support

In libyang v1 the schema nodes had a backlinks member to be able to
look up dependents of the node.  SONiC depends on this to provide
functionality it uses and it needs to be exposed via the python
module.

In theory, exposing the 'dfs' functions could make this work, but
it would likely be cost prohibitive since walking the tree would
be expensive to create a python node for evaluation in native
python.

This implementation provides 2 python functions:
 * Context.backlinks_find_leafref_nodes(path) - This function can
   take the path of the base node and find all dependents.  If
   no path is specified, then it will return all nodes that contain
   a leafref reference.
 * Context.backlinks_xpath_leafrefs(xpath) - This function takes
   an xpath, then returns all target nodes the xpath may reference.
   Typically only one will be returned, but multiples may be in the
   case of a union.

A user can build a cache by combining Context.backlinks_find_leafref_nodes()
with no path set and building a reverse table using
Context.backlinks_xpath_leafrefs(xpath)

Signed-off-by: Brad House <[email protected]>

cffi/cdefs.h

@@ -1350,3 +1350,9 @@ extern "Python" void lypy_lyplg_ext_compile_free_clb(const struct ly_ctx *, stru
 
 /* from libc, needed to free allocated strings */
 void free(void *);
+
+/* From source.c custom C code helpers for backlinks */
+size_t pyly_backlinks_xpath_leafrefs(const struct ly_ctx *ctx, const char *xpath, char ***out);
+size_t pyly_backlinks_find_leafref_nodes(const struct ly_ctx *ctx, const char *base_path, int include_children, char ***out);
+
+void pyly_cstr_array_free(char **list, size_t nlist);

cffi/source.c

@@ -9,3 +9,260 @@
 #if (LY_VERSION_MAJOR != 3)
 #error "This version of libyang bindings only works with libyang 3.x"
 #endif
+
+typedef struct {
+    char **results;
+    size_t nresults;
+    size_t alloc_results;
+} pyly_string_list_t;
+
+/*! Takes append an entry to a dynamic array of strings
+ *  \param[in] l    Pointer to pyly_string_list_t object (must be initialized to zero)
+ *  \param[in] str  String, the pointer will be owned by the list
+ */
+static void pyly_strlist_append(pyly_string_list_t *l, char *str /* Takes ownership */)
+{
+    if (l == NULL || str == NULL) {
+        return;
+    }
+
+    if (l->nresults + 1 > l->alloc_results) {
+        if (l->alloc_results == 0) {
+            l->alloc_results = 1;
+        } else {
+            l->alloc_results <<= 1;
+        }
+        l->results = realloc(l->results, l->alloc_results * sizeof(*l->results));
+    }
+    l->results[l->nresults++] = str;
+}
+
+void pyly_cstr_array_free(char **list, size_t nlist)
+{
+    size_t i;
+
+    if (list == NULL)
+        return;
+
+    for (i=0; i<nlist; i++) {
+        free(list[i]);
+    }
+    free(list);
+}
+
+typedef struct {
+    const struct ly_ctx *ctx;
+    const char          *base_path;
+    int                  include_children;
+    pyly_string_list_t  *res;
+} pyly_dfs_data_t;
+
+static char *pyly_lref_to_xpath(const struct lysc_node *node, const struct lysc_type_leafref *lref)
+{
+    struct ly_set *set = NULL;
+    LY_ERR         err;
+    char          *path = NULL;
+
+    err = lys_find_expr_atoms(node, node->module, lref->path, lref->prefixes, 0, &set);
+    if (err != LY_SUCCESS) {
+        return NULL;
+    }
+
+    if (set->count != 0) {
+        path = lysc_path(set->snodes[set->count - 1], LYSC_PATH_DATA, NULL, 0);
+    }
+
+    ly_set_free(set, NULL);
+    return path;
+}
+
+static size_t pyly_snode_fetch_leafrefs(const struct lysc_node *node, char ***out)
+{
+    pyly_string_list_t           res;
+    const struct lysc_node_leaf *leaf;
+
+    if (node == NULL || out == NULL) {
+        return 0;
+    }
+
+    memset(&res, 0, sizeof(res));
+    *out = NULL;
+
+    /* Not a node type we are interested in */
+    if (node->nodetype != LYS_LEAF && node->nodetype != LYS_LEAFLIST) {
+        return 0;
+    }
+
+    leaf = (const struct lysc_node_leaf *)node;
+    if (leaf->type->basetype == LY_TYPE_UNION) {
+        /* Unions are a bit of a pain as they aren't represented by nodes,
+         * so we need to iterate across them to see if they contain any
+         * leafrefs */
+        const struct lysc_type_union *un = (const struct lysc_type_union *)leaf->type;
+        size_t                        i;
+
+        for (i=0; i<LY_ARRAY_COUNT(un->types); i++) {
+            const struct lysc_type *utype = un->types[i];
+
+            if (utype->basetype != LY_TYPE_LEAFREF) {
+                continue;
+            }
+
+            pyly_strlist_append(&res, pyly_lref_to_xpath(node, (const struct lysc_type_leafref *)utype));
+        }
+    } else if (leaf->type->basetype == LY_TYPE_LEAFREF) {
+        const struct lysc_node *base_node = lysc_node_lref_target(node);
+
+        if (base_node == NULL) {
+            return 0;
+        }
+
+        pyly_strlist_append(&res, lysc_path(base_node, LYSC_PATH_DATA, NULL, 0));
+    } else {
+        /* Not a node type we're interested in */
+        return 0;
+    }
+
+    *out = res.results;
+    return res.nresults;
+}
+
+/*! For the given xpath, return the xpaths for the nodes they reference.
+ *
+ *  \param[in]  ctx       Initialized context with loaded schema
+ *  \param[in]  xpath     Xpath
+ *  \param[out] out       Pointer passed by reference that will hold a C array
+ *                        of c strings representing the paths for any leaf
+ *                        references.
+ *  \return number of results, or 0 if none.
+ */
+size_t pyly_backlinks_xpath_leafrefs(const struct ly_ctx *ctx, const char *xpath, char ***out)
+{
+    LY_ERR             err;
+    struct ly_set     *set = NULL;
+    size_t             i;
+    pyly_string_list_t res;
+
+    if (ctx == NULL || xpath == NULL || out == NULL) {
+        return 0;
+    }
+
+    memset(&res, 0, sizeof(res));
+
+    *out = NULL;
+
+    err = lys_find_xpath(ctx, NULL, xpath, 0, &set);
+    if (err != LY_SUCCESS) {
+        return 0;
+    }
+
+    for (i=0; i<set->count; i++) {
+        size_t cnt;
+        size_t j;
+        char **refs = NULL;
+        cnt = pyly_snode_fetch_leafrefs(set->snodes[i], &refs);
+        for (j=0; j<cnt; j++) {
+            pyly_strlist_append(&res, strdup(refs[j]));
+        }
+        pyly_cstr_array_free(refs, cnt);
+    }
+
+    ly_set_free(set, NULL);
+
+    *out = res.results;
+    return res.nresults;
+}
+
+static LY_ERR pyly_backlinks_find_leafref_nodes_clb(struct lysc_node *node, void *data, ly_bool *dfs_continue)
+{
+    pyly_dfs_data_t *dctx     = data;
+    char           **leafrefs = NULL;
+    size_t           nleafrefs;
+    size_t           i;
+    int              found    = 0;
+
+    /* Not a node type we are interested in */
+    if (node->nodetype != LYS_LEAF && node->nodetype != LYS_LEAFLIST) {
+        return LY_SUCCESS;
+    }
+
+    /* Fetch leafrefs for comparison against our base path.  Even if we are
+     * going to throw them away, we still need a count to know if this has
+     * leafrefs */
+    nleafrefs = pyly_snode_fetch_leafrefs(node, &leafrefs);
+    if (nleafrefs == 0) {
+        return LY_SUCCESS;
+    }
+
+    for (i=0; i<nleafrefs && !found; i++) {
+        if (dctx->base_path != NULL) {
+            if (dctx->include_children) {
+                if (strncmp(leafrefs[i], dctx->base_path, strlen(dctx->base_path)) != 0) {
+                    continue;
+                }
+            } else {
+                if (strcmp(leafrefs[i], dctx->base_path) != 0) {
+                    continue;
+                }
+            }
+        }
+        found = 1;
+    }
+    pyly_cstr_array_free(leafrefs, nleafrefs);
+
+    if (found) {
+        pyly_strlist_append(dctx->res, lysc_path(node, LYSC_PATH_DATA, NULL, 0));
+    }
+
+    return LY_SUCCESS;
+}
+
+/*! Search the entire loaded schema for any nodes that contain a leafref and
+ *  record the path.  If a base_path is specified, only leafrefs that point to
+ *  the specified path will be recorded, if include_children is 1, then children
+ *  of the specified path are also included.
+ *
+ *  This function is used in replacement for the concept of backlink references
+ *  that were part of libyang v1 but were subsequently removed.  This is
+ *  implemented in C code due to the overhead involved with needing to produce
+ *  Python nodes for results for evaluation which has a high overhead.
+ *
+ *  If building a data cache to more accurately replicate the prior backlinks
+ *  concept, pass NULL to base_path which will then return any paths that
+ *  reference another.  It is then the caller's responsibility to look up where
+ *  the leafref is pointing as part of building the cache. It is expected most
+ *  users will not need the cache and will simply pass in the base_path as needed.
+ *
+ *  \param[in]  ctx              Initialized context with loaded schema
+ *  \param[in]  base_path        Optional base node path to restrict output.
+ *  \param[in]  include_children Whether or not to include children of the
+ *                               specified base path or if the path is an
+ *                               explicit reference.
+ *  \param[out] out              Pointer passed by reference that will hold a C
+ *                               array of c strings representing the paths for
+ *                               any leaf references.
+ *  \return number of results, or 0 if none.
+ */
+size_t pyly_backlinks_find_leafref_nodes(const struct ly_ctx *ctx, const char *base_path, int include_children, char ***out)
+{
+    pyly_string_list_t       res;
+    uint32_t                 module_idx = 0;
+    const struct lys_module *module;
+
+    memset(&res, 0, sizeof(res));
+
+    if (ctx == NULL || out == NULL) {
+        return 0;
+    }
+
+    /* Iterate across all loaded modules */
+    for (module_idx = 0; (module = ly_ctx_get_module_iter(ctx, &module_idx)) != NULL; ) {
+        pyly_dfs_data_t data = { ctx, base_path, include_children, &res };
+
+        lysc_module_dfs_full(module, pyly_backlinks_find_leafref_nodes_clb, &data);
+        /* Ignore error */
+    }
+
+    *out = res.results;
+    return res.nresults;
+}

libyang/context.py

@@ -646,6 +646,42 @@ def parse_data_file(
             json_null=json_null,
         )
 
+    def backlinks_find_leafref_nodes(self, base_path: str = None, include_children: bool = False) -> list[str]:
+        if self.cdata is None:
+            raise RuntimeError("context already destroyed")
+
+        out = []
+
+        carray = ffi.new("char ***")
+        clen = lib.pyly_backlinks_find_leafref_nodes(
+            self.cdata, str2c(base_path), 1 if include_children else 0, carray
+        )
+        if clen == 0:
+            return out
+
+        for i in range(clen):
+            out.append(c2str(carray[0][i]))
+
+        lib.pyly_cstr_array_free(carray[0], clen)
+        return out
+
+    def backlinks_xpath_leafrefs(self, xpath: str) -> list[str]:
+        if self.cdata is None:
+            raise RuntimeError("context already destroyed")
+
+        out = []
+
+        carray = ffi.new("char ***")
+        clen = lib.pyly_backlinks_xpath_leafrefs(self.cdata, str2c(xpath), carray)
+        if clen == 0:
+            return out
+
+        for i in range(clen):
+            out.append(c2str(carray[0][i]))
+
+        lib.pyly_cstr_array_free(carray[0], clen)
+        return out
+
     def __iter__(self) -> Iterator[Module]:
         """
         Return an iterator that yields all implemented modules from the context

tests/test_schema.py

@@ -801,6 +801,73 @@ def test_leaf_list_parsed(self):
         self.assertFalse(pnode.ordered())
 
 
+# -------------------------------------------------------------------------------------
+class BacklinksTest(unittest.TestCase):
+    def setUp(self):
+        self.ctx = Context(YANG_DIR)
+        self.ctx.load_module("yolo-leafref-search")
+        self.ctx.load_module("yolo-leafref-search-extmod")
+
+    def tearDown(self):
+        self.ctx.destroy()
+        self.ctx = None
+
+    def test_backlinks_all_nodes(self):
+        expected = [
+            "/yolo-leafref-search-extmod:my_extref_list/my_extref",
+            "/yolo-leafref-search:refstr",
+            "/yolo-leafref-search:refnum",
+            "/yolo-leafref-search-extmod:my_extref_list/my_extref_union"
+        ]
+        refs = self.ctx.backlinks_find_leafref_nodes()
+
+        expected.sort()
+        refs.sort()
+        self.assertEqual(expected, refs)
+
+    def test_backlinks_one(self):
+        expected = [
+            "/yolo-leafref-search-extmod:my_extref_list/my_extref",
+            "/yolo-leafref-search:refstr",
+            "/yolo-leafref-search-extmod:my_extref_list/my_extref_union"
+        ]
+        refs = self.ctx.backlinks_find_leafref_nodes(
+            base_path="/yolo-leafref-search:my_list/my_leaf_string"
+        )
+
+        expected.sort()
+        refs.sort()
+        self.assertEqual(expected, refs)
+
+    def test_backlinks_children(self):
+        expected = [
+            "/yolo-leafref-search-extmod:my_extref_list/my_extref",
+            "/yolo-leafref-search:refstr",
+            "/yolo-leafref-search:refnum",
+            "/yolo-leafref-search-extmod:my_extref_list/my_extref_union"
+        ]
+        refs = self.ctx.backlinks_find_leafref_nodes(
+            base_path="/yolo-leafref-search:my_list/",
+            include_children=True
+        )
+
+        expected.sort()
+        refs.sort()
+        self.assertEqual(expected, refs)
+
+    def test_backlinks_xpath_leafrefs(self):
+        expected = [
+            "/yolo-leafref-search:my_list/my_leaf_string"
+        ]
+        refs = self.ctx.backlinks_xpath_leafrefs(
+            "/yolo-leafref-search-extmod:my_extref_list/my_extref"
+        )
+
+        expected.sort()
+        refs.sort()
+        self.assertEqual(expected, refs)
+
+
 # -------------------------------------------------------------------------------------
 class ChoiceTest(unittest.TestCase):
     def setUp(self):