Skip to content

feat(gist): fsspec file system for GitHub gists (resolves #888) #1791

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 23, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/source/api.rst
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ Built-in Implementations
fsspec.implementations.dbfs.DatabricksFileSystem
fsspec.implementations.dirfs.DirFileSystem
fsspec.implementations.ftp.FTPFileSystem
fsspec.implementations.gist.GistFileSystem
fsspec.implementations.git.GitFileSystem
fsspec.implementations.github.GithubFileSystem
fsspec.implementations.http.HTTPFileSystem
Expand Down Expand Up @@ -162,6 +163,9 @@ Built-in Implementations
.. autoclass:: fsspec.implementations.ftp.FTPFileSystem
:members: __init__

.. autoclass:: fsspec.implementations.gist.GistFileSystem
:members: __init__

.. autoclass:: fsspec.implementations.git.GitFileSystem
:members: __init__

Expand Down
232 changes: 232 additions & 0 deletions fsspec/implementations/gist.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,232 @@
import requests

from ..spec import AbstractFileSystem
from ..utils import infer_storage_options
from .memory import MemoryFile


class GistFileSystem(AbstractFileSystem):
"""
Interface to files in a single GitHub Gist.

Provides read-only access to a gist's files. Gists do not contain
subdirectories, so file listing is straightforward.

Parameters
----------
gist_id : str
The ID of the gist you want to access (the long hex value from the URL).
filenames : list[str] (optional)
If provided, only make a file system representing these files, and do not fetch
the list of all files for this gist.
sha : str (optional)
If provided, fetch a particular revision of the gist. If omitted,
the latest revision is used.
username : str (optional)
GitHub username for authentication (required if token is given).
token : str (optional)
GitHub personal access token (required if username is given).
timeout : (float, float) or float, optional
Connect and read timeouts for requests (default 60s each).
kwargs : dict
Stored on `self.request_kw` and passed to `requests.get` when fetching Gist
metadata or reading ("opening") a file.
"""

protocol = "gist"
gist_url = "https://api.github.com/gists/{gist_id}"
gist_rev_url = "https://api.github.com/gists/{gist_id}/{sha}"

def __init__(
self,
gist_id,
filenames=None,
sha=None,
username=None,
token=None,
timeout=None,
**kwargs,
):
super().__init__()
self.gist_id = gist_id
self.filenames = filenames
self.sha = sha # revision of the gist (optional)
if (username is None) ^ (token is None):
# Both or neither must be set
if username or token:
raise ValueError("Auth requires both username and token, or neither.")
self.username = username
self.token = token
self.request_kw = kwargs
# Default timeouts to 60s connect/read if none provided
self.timeout = timeout if timeout is not None else (60, 60)

# We use a single-level "directory" cache, because a gist is essentially flat
self.dircache[""] = self._fetch_file_list()

@property
def kw(self):
"""Auth parameters passed to 'requests' if we have username/token."""
if self.username is not None and self.token is not None:
return {"auth": (self.username, self.token), **self.request_kw}
return self.request_kw

def _fetch_gist_metadata(self):
"""
Fetch the JSON metadata for this gist (possibly for a specific revision).
"""
if self.sha:
url = self.gist_rev_url.format(gist_id=self.gist_id, sha=self.sha)
else:
url = self.gist_url.format(gist_id=self.gist_id)

r = requests.get(url, timeout=self.timeout, **self.kw)
if r.status_code == 404:
raise FileNotFoundError(
f"Gist not found: {self.gist_id}@{self.sha or 'latest'}"
)
r.raise_for_status()
return r.json()

def _fetch_file_list(self):
"""
Returns a list of dicts describing each file in the gist. These get stored
in self.dircache[""].
"""
meta = self._fetch_gist_metadata()
if self.filenames:
available_files = meta.get("files", {})
files = {}
for fn in self.filenames:
if fn not in available_files:
raise FileNotFoundError(fn)
files[fn] = available_files[fn]
else:
files = meta.get("files", {})

out = []
for fname, finfo in files.items():
if finfo is None:
# Occasionally GitHub returns a file entry with null if it was deleted
continue
# Build a directory entry
out.append(
{
"name": fname, # file's name
"type": "file", # gists have no subdirectories
"size": finfo.get("size", 0), # file size in bytes
"raw_url": finfo.get("raw_url"),
}
)
return out

@classmethod
def _strip_protocol(cls, path):
"""
Remove 'gist://' from the path, if present.
"""
# The default infer_storage_options can handle gist://username:token@id/file
# or gist://id/file, but let's ensure we handle a normal usage too.
# We'll just strip the protocol prefix if it exists.
path = infer_storage_options(path).get("path", path)
return path.lstrip("/")

@staticmethod
def _get_kwargs_from_urls(path):
"""
Parse 'gist://' style URLs into GistFileSystem constructor kwargs.
For example:
gist://:TOKEN@<gist_id>/file.txt
gist://username:TOKEN@<gist_id>/file.txt
"""
so = infer_storage_options(path)
out = {}
if "username" in so and so["username"]:
out["username"] = so["username"]
if "password" in so and so["password"]:
out["token"] = so["password"]
if "host" in so and so["host"]:
# We interpret 'host' as the gist ID
out["gist_id"] = so["host"]

# Extract SHA and filename from path
if "path" in so and so["path"]:
path_parts = so["path"].rsplit("/", 2)[-2:]
if len(path_parts) == 2:
if path_parts[0]: # SHA present
out["sha"] = path_parts[0]
if path_parts[1]: # filename also present
out["filenames"] = [path_parts[1]]

return out

def ls(self, path="", detail=False, **kwargs):
"""
List files in the gist. Gists are single-level, so any 'path' is basically
the filename, or empty for all files.

Parameters
----------
path : str, optional
The filename to list. If empty, returns all files in the gist.
detail : bool, default False
If True, return a list of dicts; if False, return a list of filenames.
"""
path = self._strip_protocol(path or "")
# If path is empty, return all
if path == "":
results = self.dircache[""]
else:
# We want just the single file with this name
all_files = self.dircache[""]
results = [f for f in all_files if f["name"] == path]
if not results:
raise FileNotFoundError(path)
if detail:
return results
else:
return sorted(f["name"] for f in results)

def _open(self, path, mode="rb", block_size=None, **kwargs):
"""
Read a single file from the gist.
"""
if mode != "rb":
raise NotImplementedError("GitHub Gist FS is read-only (no write).")

path = self._strip_protocol(path)
# Find the file entry in our dircache
matches = [f for f in self.dircache[""] if f["name"] == path]
if not matches:
raise FileNotFoundError(path)
finfo = matches[0]

raw_url = finfo.get("raw_url")
if not raw_url:
raise FileNotFoundError(f"No raw_url for file: {path}")

r = requests.get(raw_url, timeout=self.timeout, **self.kw)
if r.status_code == 404:
raise FileNotFoundError(path)
r.raise_for_status()
return MemoryFile(path, None, r.content)

def cat(self, path, recursive=False, on_error="raise", **kwargs):
"""
Return {path: contents} for the given file or files. If 'recursive' is True,
and path is empty, returns all files in the gist.
"""
paths = self.expand_path(path, recursive=recursive)
out = {}
for p in paths:
try:
with self.open(p, "rb") as f:
out[p] = f.read()
except FileNotFoundError as e:
if on_error == "raise":
raise e
elif on_error == "omit":
pass # skip
else:
out[p] = e
return out
2 changes: 0 additions & 2 deletions fsspec/implementations/github.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
from ..utils import infer_storage_options
from .memory import MemoryFile

# TODO: add GIST backend, would be very similar


class GithubFileSystem(AbstractFileSystem):
"""Interface to files in github
Expand Down
82 changes: 82 additions & 0 deletions fsspec/implementations/tests/test_gist.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
import pytest

import fsspec
from fsspec.implementations.gist import GistFileSystem


@pytest.mark.parametrize(
"gist_id,sha",
[("2656908684d3965b80c2", "2fb2f12f332f7e242b1a2af1f41e30ddf99f24c7")],
)
def test_gist_public_all_files(gist_id, sha):
fs = fsspec.filesystem("gist", gist_id=gist_id, sha=sha)
# Listing
all_files = fs.ls("")
assert len(all_files) == 2
# Cat
data = fs.cat(all_files)
assert set(data.keys()) == set(all_files)
for v in data.values():
assert isinstance(v, bytes)


@pytest.mark.parametrize(
"gist_id,sha,file",
[
(
"2656908684d3965b80c2",
"2fb2f12f332f7e242b1a2af1f41e30ddf99f24c7",
"distributed_error_logs_PY3_7-3-2016",
)
],
)
def test_gist_public_one_file(gist_id, sha, file):
fs = fsspec.filesystem("gist", gist_id=gist_id, sha=sha, filenames=[file])
# Listing
all_files = fs.ls("")
assert len(all_files) == 1
# Cat
data = fs.cat(all_files)
assert set(data.keys()) == set(all_files)
for v in data.values():
assert isinstance(v, bytes)


@pytest.mark.parametrize(
"gist_id,sha,file",
[
(
"2656908684d3965b80c2",
"2fb2f12f332f7e242b1a2af1f41e30ddf99f24c7",
"file-that-doesnt-exist.py",
)
],
)
def test_gist_public_missing_file(gist_id, sha, file):
with pytest.raises(FileNotFoundError):
fsspec.filesystem("gist", gist_id=gist_id, sha=sha, filenames=[file])


@pytest.mark.parametrize(
"gist_id,sha,file,token,user",
[
("gist-id-123", "sha_hash_a0b1", "a_file.txt", "secret_token", "my-user"),
("gist-id-123", "sha_hash_a0b1", "a_file.txt", "secret_token", ""), # No user
("gist-id-123", "", "a_file.txt", "secret_token", "my-user"), # No SHA
],
)
def test_gist_url_parse(gist_id, sha, file, token, user):
if sha:
fmt_str = f"gist://{user}:{token}@{gist_id}/{sha}/{file}"
else:
fmt_str = f"gist://{user}:{token}@{gist_id}/{file}"

parsed = GistFileSystem._get_kwargs_from_urls(fmt_str)

expected = {"gist_id": gist_id, "token": token, "filenames": [file]}
if user: # Only include username if it's not empty
expected["username"] = user
if sha: # Only include SHA if it's specified
expected["sha"] = sha

assert parsed == expected
4 changes: 4 additions & 0 deletions fsspec/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,10 @@ def register_implementation(name, cls, clobber=False, errtxt=None):
"err": "Please install gdrivefs for access to Google Drive",
},
"generic": {"class": "fsspec.generic.GenericFileSystem"},
"gist": {
"class": "fsspec.implementations.gist.GistFileSystem",
"err": "Install the requests package to use the gist FS",
},
"git": {
"class": "fsspec.implementations.git.GitFileSystem",
"err": "Install pygit2 to browse local git repos",
Expand Down
Loading