Boost references speed (fsspec#892)

* Boost references speed by skipping _ls_from_cache

* Don't process templates if there are none

* Don't make dircache until needed

(zarr access doesn't need it!)

Author: martindurant

fsspec/implementations/github.py

@@ -156,15 +156,17 @@ def ls(self, path, detail=False, sha=None, _sha=None, **kwargs):
             if r.status_code == 404:
                 raise FileNotFoundError(path)
             r.raise_for_status()
+            types = {"blob": "file", "tree": "directory"}
             out = [
                 {
                     "name": path + "/" + f["path"] if path else f["path"],
                     "mode": f["mode"],
-                    "type": {"blob": "file", "tree": "directory"}[f["type"]],
+                    "type": types[f["type"]],
                     "size": f.get("size", 0),
                     "sha": f["sha"],
                 }
                 for f in r.json()["tree"]
+                if f["type"] in types
             ]
             if sha in [self.root, None]:
                 self.dircache[path] = out

fsspec/implementations/reference.py

@@ -221,6 +221,7 @@ def cat_file(self, path, start=None, end=None, **kwargs):
         part_or_url, start0, end0 = self._cat_common(path)
         if isinstance(part_or_url, bytes):
             return part_or_url[start:end]
+        # TODO: update start0, end0 if start/end given, instead of slicing
         return self.fs.cat_file(part_or_url, start=start0, end=end0)[start:end]
 
     def pipe_file(self, path, value, **_):
@@ -279,7 +280,6 @@ def _render_jinja(url):
 
         if self.templates:
             self.df["url"] = self.df["url"].map(_render_jinja)
-        self._dircache_from_items()
 
     def _process_references(self, references, template_overrides=None):
         if isinstance(references, (str, bytes)):
@@ -293,7 +293,6 @@ def _process_references(self, references, template_overrides=None):
             raise ValueError(f"Unknown reference spec version: {vers}")
         # TODO: we make dircache by iterating over all entries, but for Spec >= 1,
         #  can replace with programmatic. Is it even needed for mapper interface?
-        self._dircache_from_items()
 
     def _process_references0(self, references):
         """Make reference dict for Spec Version 0"""
@@ -320,7 +319,7 @@ def _render_jinja(u):
                 if v.startswith("base64:"):
                     self.references[k] = base64.b64decode(v[7:])
                 self.references[k] = v
-            else:
+            elif self.templates:
                 u = v[0]
                 if "{{" in u:
                     if self.simple_templates:
@@ -332,6 +331,8 @@ def _render_jinja(u):
                     else:
                         u = _render_jinja(u)
                 self.references[k] = [u] if len(v) == 1 else [u, v[1], v[2]]
+            else:
+                self.references[k] = v
         self.references.update(self._process_gen(references.get("gen", [])))
 
     def _process_templates(self, tmp):
@@ -422,6 +423,8 @@ def open(self, path, mode="rb", block_size=None, cache_options=None, **kwargs):
 
     def ls(self, path, detail=True, **kwargs):
         path = self._strip_protocol(path)
+        if not self.dircache:
+            self._dircache_from_items()
         out = self._ls_from_cache(path)
         if out is None:
             raise FileNotFoundError
@@ -430,16 +433,13 @@ def ls(self, path, detail=True, **kwargs):
         return [o["name"] for o in out]
 
     def exists(self, path, **kwargs):  # overwrite auto-sync version
-        try:
-            return self._ls_from_cache(path) is not None
-        except FileNotFoundError:
-            return False
+        return self.isdir(path) or self.isfile(path)
 
     def isdir(self, path):  # overwrite auto-sync version
-        return self.exists(path) and self.info(path)["type"] == "directory"
+        return path in self.dircache
 
     def isfile(self, path):  # overwrite auto-sync version
-        return self.exists(path) and self.info(path)["type"] == "file"
+        return path in self.references
 
     async def _ls(self, path, detail=True, **kwargs):  # calls fast sync code
         return self.ls(path, detail, **kwargs)