import base64 import io import itertools import logging import os from functools import lru_cache import fsspec.core try: import ujson as json except ImportError: import json from ..asyn import AsyncFileSystem, sync from ..callbacks import _DEFAULT_CALLBACK from ..core import filesystem, open, split_protocol from ..spec import AbstractFileSystem from ..utils import isfilelike, merge_offset_ranges logger = logging.getLogger("fsspec.reference") def _first(d): return list(d.values())[0] def _prot_in_references(path, references): ref = references.get(path) if isinstance(ref, (list, tuple)): return split_protocol(ref[0])[0] if ref[0] else ref[0] def _protocol_groups(paths, references): if isinstance(paths, str): return {_prot_in_references(paths, references): [paths]} out = {} for path in paths: protocol = _prot_in_references(path, references) out.setdefault(protocol, []).append(path) return out class ReferenceFileSystem(AsyncFileSystem): """View byte ranges of some other file as a file system Initial version: single file system target, which must support async, and must allow start and end args in _cat_file. Later versions may allow multiple arbitrary URLs for the targets. This FileSystem is read-only. It is designed to be used with async targets (for now). This FileSystem only allows whole-file access, no ``open``. We do not get original file details from the target FS. Configuration is by passing a dict of references at init, or a URL to a JSON file containing the same; this dict can also contain concrete data for some set of paths. Reference dict format: {path0: bytes_data, path1: (target_url, offset, size)} https://github.com/fsspec/kerchunk/blob/main/README.md """ protocol = "reference" def __init__( self, fo, target=None, ref_storage_args=None, target_protocol=None, target_options=None, remote_protocol=None, remote_options=None, fs=None, template_overrides=None, simple_templates=True, max_gap=64_000, max_block=256_000_000, loop=None, **kwargs, ): """ Parameters ---------- fo : dict or str The set of references to use for this instance, with a structure as above. If str, will use fsspec.open, in conjunction with target_options and target_protocol to open and parse JSON at this location. target : str For any references having target_url as None, this is the default file target to use ref_storage_args : dict If references is a str, use these kwargs for loading the JSON file. Deprecated: use target_options instead. target_protocol : str Used for loading the reference file, if it is a path. If None, protocol will be derived from the given path target_options : dict Extra FS options for loading the reference file ``fo``, if given as a path remote_protocol : str The protocol of the filesystem on which the references will be evaluated (unless fs is provided). If not given, will be derived from the first URL that has a protocol in the templates or in the references, in that order. remote_options : dict kwargs to go with remote_protocol fs : AbstractFileSystem | dict(str, (AbstractFileSystem | dict)) Directly provide a file system(s): - a single filesystem instance - a dict of protocol:filesystem, where each value is either a filesystem instance, or a dict of kwargs that can be used to create in instance for the given protocol If this is given, remote_options and remote_protocol are ignored. template_overrides : dict Swap out any templates in the references file with these - useful for testing. simple_templates: bool Whether templates can be processed with simple replace (True) or if jinja is needed (False, much slower). All reference sets produced by ``kerchunk`` are simple in this sense, but the spec allows for complex. max_gap, max_block: int For merging multiple concurrent requests to the same remote file. Neighboring byte ranges will only be merged when their inter-range gap is <= ``max_gap``. Default is 64KB. Set to 0 to only merge when it requires no extra bytes. Pass a negative number to disable merging, appropriate for local target files. Neighboring byte ranges will only be merged when the size of the aggregated range is <= ``max_block``. Default is 256MB. kwargs : passed to parent class """ super().__init__(loop=loop, **kwargs) self.target = target self.dataframe = False self.template_overrides = template_overrides self.simple_templates = simple_templates self.templates = {} self.fss = {} self.max_gap = max_gap self.max_block = max_block if hasattr(fo, "read"): text = fo.read() elif isinstance(fo, str): if target_protocol: extra = {"protocol": target_protocol} else: extra = {} dic = dict(**(ref_storage_args or target_options or {}), **extra) # text JSON with open(fo, "rb", **dic) as f: logger.info("Read reference from URL %s", fo) text = f.read() else: # dictionaries text = fo if self.dataframe: self._process_dataframe() else: self._process_references(text, template_overrides) if isinstance(fs, dict): self.fss = { k: ( fsspec.filesystem(k.split(":", 1)[0], **opts) if isinstance(opts, dict) else opts ) for k, opts in fs.items() } if None not in self.fss: self.fss[None] = filesystem("file") return if fs is not None: # single remote FS remote_protocol = ( fs.protocol[0] if isinstance(fs.protocol, tuple) else fs.protocol ) self.fss[remote_protocol] = fs if remote_protocol is None: # get single protocol from any templates for ref in self.templates.values(): if callable(ref): ref = ref() protocol, _ = fsspec.core.split_protocol(ref) if protocol and protocol not in self.fss: fs = filesystem(protocol, loop=loop, **(remote_options or {})) self.fss[protocol] = fs if remote_protocol is None: # get single protocol from references for ref in self.references.values(): if callable(ref): ref = ref() if isinstance(ref, list) and ref[0]: protocol, _ = fsspec.core.split_protocol(ref[0]) if protocol and protocol not in self.fss: fs = filesystem(protocol, loop=loop, **(remote_options or {})) self.fss[protocol] = fs if remote_protocol and remote_protocol not in self.fss: fs = filesystem(remote_protocol, loop=loop, **(remote_options or {})) self.fss[remote_protocol] = fs self.fss[None] = fs or filesystem("file") # default one @property def loop(self): inloop = [fs.loop for fs in self.fss.values() if fs.async_impl] return inloop[0] if inloop else self._loop def _cat_common(self, path, start=None, end=None): path = self._strip_protocol(path) logger.debug(f"cat: {path}") part = self.references[path] if isinstance(part, str): part = part.encode() if isinstance(part, bytes): logger.debug(f"Reference: {path}, type bytes") if part.startswith(b"base64:"): part = base64.b64decode(part[7:]) return part, None, None if len(part) == 1: logger.debug(f"Reference: {path}, whole file") url = part[0] start1, end1 = start, end else: url, start0, size = part logger.debug(f"Reference: {path} => {url}, offset {start0}, size {size}") end0 = start0 + size if start is not None: if start >= 0: start1 = start0 + start else: start1 = end0 + start else: start1 = start0 if end is not None: if end >= 0: end1 = start0 + end else: end1 = end0 + end else: end1 = end0 if url is None: url = self.target return url, start1, end1 async def _cat_file(self, path, start=None, end=None, **kwargs): part_or_url, start0, end0 = self._cat_common(path, start=start, end=end) if isinstance(part_or_url, bytes): return part_or_url[start:end] protocol, _ = split_protocol(part_or_url) return await self.fss[protocol]._cat_file(part_or_url, start=start, end=end) def cat_file(self, path, start=None, end=None, **kwargs): part_or_url, start0, end0 = self._cat_common(path, start=start, end=end) if isinstance(part_or_url, bytes): return part_or_url[start:end] protocol, _ = split_protocol(part_or_url) # TODO: start and end should be passed to cat_file, not sliced return self.fss[protocol].cat_file(part_or_url, start=start0, end=end0) def pipe_file(self, path, value, **_): """Temporarily add binary data or reference as a file""" self.references[path] = value async def _get_file(self, rpath, lpath, **kwargs): if self.isdir(rpath): return os.makedirs(lpath, exist_ok=True) data = await self._cat_file(rpath) with open(lpath, "wb") as f: f.write(data) def get_file(self, rpath, lpath, callback=_DEFAULT_CALLBACK, **kwargs): if self.isdir(rpath): return os.makedirs(lpath, exist_ok=True) data = self.cat_file(rpath, **kwargs) callback.set_size(len(data)) if isfilelike(lpath): lpath.write(data) else: with open(lpath, "wb") as f: f.write(data) callback.absolute_update(len(data)) def get(self, rpath, lpath, recursive=False, **kwargs): if isinstance(lpath, list): # because we have to figure out here which lpath goes with which path # after grouping raise NotImplementedError proto_dict = _protocol_groups(rpath, self.references) for proto, paths in proto_dict.items(): if self.fss[proto].async_impl: sync(self.loop, self._get, paths, lpath, recursive, **kwargs) else: AbstractFileSystem.get( self, paths, lpath, recursive=recursive, **kwargs ) def cat(self, path, recursive=False, on_error="raise", **kwargs): if isinstance(path, str) and recursive: raise NotImplementedError if isinstance(path, list) and (recursive or any("*" in p for p in path)): raise NotImplementedError proto_dict = _protocol_groups(path, self.references) out = {} for proto, paths in proto_dict.items(): fs = self.fss[proto] urls, starts, ends = zip(*[self._cat_common(p) for p in paths]) urls2 = [] starts2 = [] ends2 = [] paths2 = [] whole_files = set() for u, s, e, p in zip(urls, starts, ends, paths): if isinstance(u, bytes): # data out[p] = u elif s is None: # whole file - limits are None, None, but no further # entries take for this file whole_files.add(u) urls2.append(u) starts2.append(s) ends2.append(e) paths2.append(p) for u, s, e, p in zip(urls, starts, ends, paths): if s is not None and u not in whole_files: urls2.append(u) starts2.append(s) ends2.append(e) paths2.append(p) new_paths, new_starts, new_ends = merge_offset_ranges( list(urls2), list(starts2), list(ends2), sort=True, max_gap=self.max_gap, max_block=self.max_block, ) bytes_out = fs.cat_ranges(new_paths, new_starts, new_ends) if len(urls2) == len(bytes_out): # we didn't do any merging for p, b in zip(paths2, bytes_out): out[p] = b else: # unbundle from merged bytes - simple approach for u, s, e, p in zip(urls, starts, ends, paths): if p in out: continue # was bytes, already handled for np, ns, ne, b in zip( new_paths, new_starts, new_ends, bytes_out ): if np == u and (ns is None or ne is None): out[p] = b[s:e] elif np == u and s >= ns and e <= ne: out[p] = b[s - ns : (e - ne) or None] if len(out) == 1 and isinstance(path, str) and "*" not in path: return _first(out) return out def _process_dataframe(self): self._process_templates(self.templates) @lru_cache(1000) def _render_jinja(url): if "{{" in url: if self.simple_templates: return ( url.replace("{{", "{") .replace("}}", "}") .format(**self.templates) ) import jinja2 return jinja2.Template(url).render(**self.templates) return url if self.templates: self.df["url"] = self.df["url"].map(_render_jinja) def _process_references(self, references, template_overrides=None): if isinstance(references, (str, bytes)): references = json.loads(references) vers = references.get("version", None) if vers is None: self._process_references0(references) elif vers == 1: self._process_references1(references, template_overrides=template_overrides) else: raise ValueError(f"Unknown reference spec version: {vers}") # TODO: we make dircache by iterating over all entries, but for Spec >= 1, # can replace with programmatic. Is it even needed for mapper interface? def _process_references0(self, references): """Make reference dict for Spec Version 0""" if "zarr_consolidated_format" in references: # special case for Ike prototype references = _unmodel_hdf5(references) self.references = references def _process_references1(self, references, template_overrides=None): if not self.simple_templates or self.templates: try: import jinja2 except ImportError as e: raise ValueError("Reference Spec Version 1 requires jinja2") from e self.references = {} self._process_templates(references.get("templates", {})) @lru_cache(1000) def _render_jinja(u): return jinja2.Template(u).render(**self.templates) for k, v in references.get("refs", {}).items(): if isinstance(v, str): if v.startswith("base64:"): self.references[k] = base64.b64decode(v[7:]) self.references[k] = v elif self.templates: u = v[0] if "{{" in u: if self.simple_templates: u = ( u.replace("{{", "{") .replace("}}", "}") .format(**self.templates) ) else: u = _render_jinja(u) self.references[k] = [u] if len(v) == 1 else [u, v[1], v[2]] else: self.references[k] = v self.references.update(self._process_gen(references.get("gen", []))) def _process_templates(self, tmp): self.templates = {} if self.template_overrides is not None: tmp.update(self.template_overrides) for k, v in tmp.items(): if "{{" in v: import jinja2 self.templates[k] = lambda temp=v, **kwargs: jinja2.Template( temp ).render(**kwargs) else: self.templates[k] = v def _process_gen(self, gens): out = {} for gen in gens: dimension = { k: v if isinstance(v, list) else range(v.get("start", 0), v["stop"], v.get("step", 1)) for k, v in gen["dimensions"].items() } products = ( dict(zip(dimension.keys(), values)) for values in itertools.product(*dimension.values()) ) for pr in products: import jinja2 key = jinja2.Template(gen["key"]).render(**pr, **self.templates) url = jinja2.Template(gen["url"]).render(**pr, **self.templates) if ("offset" in gen) and ("length" in gen): offset = int( jinja2.Template(gen["offset"]).render(**pr, **self.templates) ) length = int( jinja2.Template(gen["length"]).render(**pr, **self.templates) ) out[key] = [url, offset, length] elif ("offset" in gen) ^ ("length" in gen): raise ValueError( "Both 'offset' and 'length' are required for a " "reference generator entry if either is provided." ) else: out[key] = [url] return out def _dircache_from_items(self): self.dircache = {"": []} if self.dataframe: it = self.df.iterrows() else: it = self.references.items() for path, part in it: if self.dataframe: if part["data"]: size = len(part["data"]) else: size = part["size"] else: if isinstance(part, (bytes, str)): size = len(part) elif len(part) == 1: size = None else: _, start, size = part par = path.rsplit("/", 1)[0] if "/" in path else "" par0 = par while par0 and par0 not in self.dircache: # build parent directories self.dircache[par0] = [] self.dircache.setdefault( par0.rsplit("/", 1)[0] if "/" in par0 else "", [] ).append({"name": par0, "type": "directory", "size": 0}) par0 = self._parent(par0) self.dircache[par].append({"name": path, "type": "file", "size": size}) def open(self, path, mode="rb", block_size=None, cache_options=None, **kwargs): if mode != "rb": raise NotImplementedError data = self.cat_file(path) # load whole chunk into memory return io.BytesIO(data) def ls(self, path, detail=True, **kwargs): path = self._strip_protocol(path) if not self.dircache: self._dircache_from_items() out = self._ls_from_cache(path) if out is None: raise FileNotFoundError(path) if detail: return out return [o["name"] for o in out] def exists(self, path, **kwargs): # overwrite auto-sync version return self.isdir(path) or self.isfile(path) def isdir(self, path): # overwrite auto-sync version if self.dircache: return path in self.dircache else: # this may be faster than building dircache for single calls, but # by looping will be slow for many calls; could cache it? return any(_.startswith(f"{path}/") for _ in self.references) def isfile(self, path): # overwrite auto-sync version return path in self.references async def _ls(self, path, detail=True, **kwargs): # calls fast sync code return self.ls(path, detail, **kwargs) def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): # TODO: details if withdirs: return super().find( path, maxdepth=maxdepth, withdirs=withdirs, detail=detail, **kwargs ) if path: path = self._strip_protocol(path) r = sorted(k for k in self.references if k.startswith(path)) else: r = sorted(self.references) if detail: if not self.dircache: self._dircache_from_items() return {k: self._ls_from_cache(k) for k in r} else: return r def info(self, path, **kwargs): if path in self.references: out = self.references[path] if isinstance(out, (str, bytes)): # decode base64 here return {"name": path, "type": "file", "size": len(out)} elif len(out) > 1: return {"name": path, "type": "file", "size": out[2]} else: out0 = [{"name": path, "type": "file", "size": None}] else: out = self.ls(path, True) out0 = [o for o in out if o["name"] == path] if not out0: return {"name": path, "type": "directory", "size": 0} if out0[0]["size"] is None: # if this is a whole remote file, update size using remote FS prot, _ = split_protocol(self.references[path][0]) out0[0]["size"] = self.fss[prot].size(self.references[path][0]) return out0[0] async def _info(self, path, **kwargs): # calls fast sync code return self.info(path) async def _rm_file(self, path, **kwargs): self.references.pop( path, None ) # ignores FileNotFound, just as well for directories self.dircache.clear() # this is a bit heavy handed async def _pipe_file(self, path, data): # can be str or bytes self.references[path] = data self.dircache.clear() # this is a bit heavy handed async def _put_file(self, lpath, rpath): # puts binary with open(lpath, "rb") as f: self.references[rpath] = f.read() self.dircache.clear() # this is a bit heavy handed def save_json(self, url, **storage_options): """Write modified references into new location""" out = {} for k, v in self.references.items(): if isinstance(v, bytes): try: out[k] = v.decode("ascii") except UnicodeDecodeError: out[k] = (b"base64:" + base64.b64encode(v)).decode() else: out[k] = v with fsspec.open(url, "wb", **storage_options) as f: f.write(json.dumps({"version": 1, "refs": out}).encode()) def _unmodel_hdf5(references): """Special JSON format from HDF5 prototype""" # see https://gist.github.com/ajelenak/80354a95b449cedea5cca508004f97a9 import re ref = {} for key, value in references["metadata"].items(): if key.endswith(".zchunkstore"): source = value.pop("source")["uri"] match = re.findall(r"https://([^.]+)\.s3\.amazonaws\.com", source) if match: source = source.replace( f"https://{match[0]}.s3.amazonaws.com", match[0] ) for k, v in value.items(): ref[k] = (source, v["offset"], v["offset"] + v["size"]) else: ref[key] = json.dumps(value).encode() return ref