From 3625147b396b5eabe84be8f1cf5fdef03256b2f0 Mon Sep 17 00:00:00 2001 From: Yehyun Choi Date: Fri, 25 Jul 2025 23:04:26 -0400 Subject: [PATCH 01/10] Initial commit for rootfilespec integration --- src/uproot/reading.py | 66 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/src/uproot/reading.py b/src/uproot/reading.py index 2c12f748c..772875d9e 100644 --- a/src/uproot/reading.py +++ b/src/uproot/reading.py @@ -553,6 +553,52 @@ def __init__( self._streamers = None self._streamer_rules = None + import types + + from rootfilespec.buffer import ReadBuffer + from rootfilespec.bootstrap import ROOTFile + from rootfilespec.dynamic import streamerinfo_to_classes + from rootfilespec.dispatch import DICTIONARY + + initial_read_size = 512 + path = Path(file_path) + with path.open("rb") as filehandle: + + def fetch_data(seek: int, size: int): + filehandle.seek(seek) + return ReadBuffer(memoryview(filehandle.read(size)), seek, 0) + + buffer = fetch_data(0, initial_read_size) + file, _ = ROOTFile.read(buffer) + + # Read root directory object, which should be contained in the initial buffer + def fetch_cached(seek: int, size: int): + if seek + size <= len(buffer): + return buffer[seek : seek + size] + msg = "Didn't find data in initial read buffer" + raise ValueError(msg) + +# rootdir = file.get_TFile(fetch_cached).rootdir + + # List to collect NotImplementedError messages + failures: list[str] = [] + + def fail_cb(_: bytes, ex: NotImplementedError): + print(f"NotImplementedError: {ex}") + failures.append(str(ex)) + + # Read all StreamerInfo (class definitions) from the file + streamerinfo = file.get_StreamerInfo(fetch_data) + + # Render the class definitions into python code + classes = streamerinfo_to_classes(streamerinfo) + + # Evaluate the python code to create the classes and add them to the DICTIONARY + oldkeys = set(DICTIONARY) + module = types.ModuleType(f"rootfilespec.generated_{id(streamerinfo)}") + sys.modules[module.__name__] = module + exec(classes, module.__dict__) + self.hook_before_create_source() source_cls, file_path = uproot._util.file_path_to_source_class( @@ -2513,9 +2559,27 @@ def get(self): if re.match(r"(std\s*::\s*)?string", self._fClassName): return cursor.string(chunk, context) - cls = self._file.class_named(self._fClassName) + from rootfilespec.dispatch import DICTIONARY + from rootfilespec.bootstrap.uproot import UprootModelAdapter + from rootfilespec.buffer import ReadBuffer try: + # construct a rootfilespec class from the dictionary + fClassName = self._fClassName + + uproot_cls = self._file.class_named(self._fClassName) + + cls = DICTIONARY[fClassName] + + # construct a ReadBuffer from the chunk data + buf = ReadBuffer(memoryview(chunk.raw_data), chunk.start, -start_cursor.origin) + obj, buf = cls.read(buf) + + # adapt the rootfilespec class to uproot's + out = UprootModelAdapter(obj) + + except KeyError as e: + out = cls.read(chunk, cursor, context, self._file, selffile, parent) except uproot.deserialization.DeserializationError: From 175654d690e495be5c0f6d81adf79e91fe477236 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 26 Jul 2025 03:16:36 +0000 Subject: [PATCH 02/10] style: pre-commit fixes --- src/uproot/reading.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/uproot/reading.py b/src/uproot/reading.py index 772875d9e..c82eabf00 100644 --- a/src/uproot/reading.py +++ b/src/uproot/reading.py @@ -555,10 +555,10 @@ def __init__( import types - from rootfilespec.buffer import ReadBuffer from rootfilespec.bootstrap import ROOTFile - from rootfilespec.dynamic import streamerinfo_to_classes + from rootfilespec.buffer import ReadBuffer from rootfilespec.dispatch import DICTIONARY + from rootfilespec.dynamic import streamerinfo_to_classes initial_read_size = 512 path = Path(file_path) @@ -578,7 +578,7 @@ def fetch_cached(seek: int, size: int): msg = "Didn't find data in initial read buffer" raise ValueError(msg) -# rootdir = file.get_TFile(fetch_cached).rootdir + # rootdir = file.get_TFile(fetch_cached).rootdir # List to collect NotImplementedError messages failures: list[str] = [] @@ -2559,9 +2559,9 @@ def get(self): if re.match(r"(std\s*::\s*)?string", self._fClassName): return cursor.string(chunk, context) - from rootfilespec.dispatch import DICTIONARY from rootfilespec.bootstrap.uproot import UprootModelAdapter from rootfilespec.buffer import ReadBuffer + from rootfilespec.dispatch import DICTIONARY try: # construct a rootfilespec class from the dictionary @@ -2572,13 +2572,15 @@ def get(self): cls = DICTIONARY[fClassName] # construct a ReadBuffer from the chunk data - buf = ReadBuffer(memoryview(chunk.raw_data), chunk.start, -start_cursor.origin) + buf = ReadBuffer( + memoryview(chunk.raw_data), chunk.start, -start_cursor.origin + ) obj, buf = cls.read(buf) # adapt the rootfilespec class to uproot's out = UprootModelAdapter(obj) - except KeyError as e: + except KeyError: out = cls.read(chunk, cursor, context, self._file, selffile, parent) From 50a86823913ddaaec3c372102972e17f4f45348c Mon Sep 17 00:00:00 2001 From: Yehyun Choi Date: Wed, 6 Aug 2025 12:17:10 -0400 Subject: [PATCH 03/10] Bugfixes --- src/uproot/reading.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/uproot/reading.py b/src/uproot/reading.py index c82eabf00..dcfa4bc45 100644 --- a/src/uproot/reading.py +++ b/src/uproot/reading.py @@ -555,10 +555,10 @@ def __init__( import types - from rootfilespec.bootstrap import ROOTFile from rootfilespec.buffer import ReadBuffer - from rootfilespec.dispatch import DICTIONARY + from rootfilespec.bootstrap import ROOTFile from rootfilespec.dynamic import streamerinfo_to_classes + from rootfilespec.dispatch import DICTIONARY initial_read_size = 512 path = Path(file_path) @@ -578,7 +578,7 @@ def fetch_cached(seek: int, size: int): msg = "Didn't find data in initial read buffer" raise ValueError(msg) - # rootdir = file.get_TFile(fetch_cached).rootdir +# rootdir = file.get_TFile(fetch_cached).rootdir # List to collect NotImplementedError messages failures: list[str] = [] @@ -2559,29 +2559,25 @@ def get(self): if re.match(r"(std\s*::\s*)?string", self._fClassName): return cursor.string(chunk, context) - from rootfilespec.bootstrap.uproot import UprootModelAdapter - from rootfilespec.buffer import ReadBuffer from rootfilespec.dispatch import DICTIONARY + from rootfilespec.bootstrap.uproot import UprootModelAdapter, create_adapter_class + from rootfilespec.buffer import ReadBuffer try: # construct a rootfilespec class from the dictionary fClassName = self._fClassName - - uproot_cls = self._file.class_named(self._fClassName) - cls = DICTIONARY[fClassName] # construct a ReadBuffer from the chunk data - buf = ReadBuffer( - memoryview(chunk.raw_data), chunk.start, -start_cursor.origin - ) + buf = ReadBuffer(memoryview(chunk.raw_data), chunk.start, -start_cursor.origin) obj, buf = cls.read(buf) # adapt the rootfilespec class to uproot's - out = UprootModelAdapter(obj) - - except KeyError: + uproot_cls = self._file.class_named(fClassName) + out = create_adapter_class(uproot_cls) + except KeyError as e: + cls = self._file.class_named(self._fClassName) out = cls.read(chunk, cursor, context, self._file, selffile, parent) except uproot.deserialization.DeserializationError: From 37b7a47688a7ce615b938065375728d36663e38f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:17:48 +0000 Subject: [PATCH 04/10] style: pre-commit fixes --- src/uproot/reading.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/uproot/reading.py b/src/uproot/reading.py index dcfa4bc45..d5ad96ad8 100644 --- a/src/uproot/reading.py +++ b/src/uproot/reading.py @@ -555,10 +555,10 @@ def __init__( import types - from rootfilespec.buffer import ReadBuffer from rootfilespec.bootstrap import ROOTFile - from rootfilespec.dynamic import streamerinfo_to_classes + from rootfilespec.buffer import ReadBuffer from rootfilespec.dispatch import DICTIONARY + from rootfilespec.dynamic import streamerinfo_to_classes initial_read_size = 512 path = Path(file_path) @@ -578,7 +578,7 @@ def fetch_cached(seek: int, size: int): msg = "Didn't find data in initial read buffer" raise ValueError(msg) -# rootdir = file.get_TFile(fetch_cached).rootdir + # rootdir = file.get_TFile(fetch_cached).rootdir # List to collect NotImplementedError messages failures: list[str] = [] @@ -2559,9 +2559,11 @@ def get(self): if re.match(r"(std\s*::\s*)?string", self._fClassName): return cursor.string(chunk, context) - from rootfilespec.dispatch import DICTIONARY - from rootfilespec.bootstrap.uproot import UprootModelAdapter, create_adapter_class + from rootfilespec.bootstrap.uproot import ( + create_adapter_class, + ) from rootfilespec.buffer import ReadBuffer + from rootfilespec.dispatch import DICTIONARY try: # construct a rootfilespec class from the dictionary @@ -2569,14 +2571,16 @@ def get(self): cls = DICTIONARY[fClassName] # construct a ReadBuffer from the chunk data - buf = ReadBuffer(memoryview(chunk.raw_data), chunk.start, -start_cursor.origin) + buf = ReadBuffer( + memoryview(chunk.raw_data), chunk.start, -start_cursor.origin + ) obj, buf = cls.read(buf) # adapt the rootfilespec class to uproot's uproot_cls = self._file.class_named(fClassName) out = create_adapter_class(uproot_cls) - except KeyError as e: + except KeyError: cls = self._file.class_named(self._fClassName) out = cls.read(chunk, cursor, context, self._file, selffile, parent) From 389013d3c2c5abfa215225bf8cdf2b5b8a8ab471 Mon Sep 17 00:00:00 2001 From: Yehyun Choi Date: Thu, 7 Aug 2025 00:48:59 -0400 Subject: [PATCH 05/10] clean up code --- src/uproot/reading.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/uproot/reading.py b/src/uproot/reading.py index d5ad96ad8..86fc98363 100644 --- a/src/uproot/reading.py +++ b/src/uproot/reading.py @@ -2566,8 +2566,12 @@ def get(self): from rootfilespec.dispatch import DICTIONARY try: - # construct a rootfilespec class from the dictionary fClassName = self._fClassName + + if fClassName not in DICTIONARY: + cls = self._file.class_named(self._fClassName) + return cls.read(chunk, cursor, context, self._file, selffile, parent) + cls = DICTIONARY[fClassName] # construct a ReadBuffer from the chunk data @@ -2576,13 +2580,9 @@ def get(self): ) obj, buf = cls.read(buf) - # adapt the rootfilespec class to uproot's uproot_cls = self._file.class_named(fClassName) - out = create_adapter_class(uproot_cls) - - except KeyError: - cls = self._file.class_named(self._fClassName) - out = cls.read(chunk, cursor, context, self._file, selffile, parent) + AdapterClass = create_adapter_class(uproot_cls) + out = AdapterClass(obj) except uproot.deserialization.DeserializationError: breadcrumbs = context.get("breadcrumbs") From 54d68bc417106fa5b78f228b1639a4677da49889 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Aug 2025 04:52:06 +0000 Subject: [PATCH 06/10] style: pre-commit fixes --- src/uproot/reading.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/uproot/reading.py b/src/uproot/reading.py index 86fc98363..c1a99377b 100644 --- a/src/uproot/reading.py +++ b/src/uproot/reading.py @@ -2570,8 +2570,10 @@ def get(self): if fClassName not in DICTIONARY: cls = self._file.class_named(self._fClassName) - return cls.read(chunk, cursor, context, self._file, selffile, parent) - + return cls.read( + chunk, cursor, context, self._file, selffile, parent + ) + cls = DICTIONARY[fClassName] # construct a ReadBuffer from the chunk data From cb81f2a59813ce8c3bc709048017d0b338fad068 Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Thu, 21 Aug 2025 09:15:40 -0500 Subject: [PATCH 07/10] Add a model adapter test --- tests/test_1474_rootfilespec.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/test_1474_rootfilespec.py diff --git a/tests/test_1474_rootfilespec.py b/tests/test_1474_rootfilespec.py new file mode 100644 index 000000000..3179567b1 --- /dev/null +++ b/tests/test_1474_rootfilespec.py @@ -0,0 +1,21 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE + +import skhep_testdata + +import uproot + + +def test_UprootModelAdapter(): + with uproot.open(skhep_testdata.data_path("uproot-mc10events.root")) as f: + blah = f["Events"] + print(blah.all_members) + print(blah.bases) + print(blah.closed) + print(blah.concrete) + print(blah.cursor) + print(blah.file) + print(blah.instance_version) + print(blah.is_memberwise) + print(blah.members) + print(blah.num_bytes) + print(blah.parent) \ No newline at end of file From ac657f0cde8b4dce08ebe27f66bb5cc558b23d15 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Aug 2025 14:16:07 +0000 Subject: [PATCH 08/10] style: pre-commit fixes --- tests/test_1474_rootfilespec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_1474_rootfilespec.py b/tests/test_1474_rootfilespec.py index 3179567b1..8620cf1d9 100644 --- a/tests/test_1474_rootfilespec.py +++ b/tests/test_1474_rootfilespec.py @@ -18,4 +18,4 @@ def test_UprootModelAdapter(): print(blah.is_memberwise) print(blah.members) print(blah.num_bytes) - print(blah.parent) \ No newline at end of file + print(blah.parent) From aaa2bd0e88730b6190897895fa8c9d305470e2dd Mon Sep 17 00:00:00 2001 From: Yehyun Choi Date: Mon, 18 Aug 2025 22:36:05 -0400 Subject: [PATCH 09/10] bugfixing regarding versioned classes --- src/uproot/reading.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/uproot/reading.py b/src/uproot/reading.py index c1a99377b..07f1a48e9 100644 --- a/src/uproot/reading.py +++ b/src/uproot/reading.py @@ -2568,11 +2568,11 @@ def get(self): try: fClassName = self._fClassName - if fClassName not in DICTIONARY: + if fClassName not in DICTIONARY: cls = self._file.class_named(self._fClassName) return cls.read( chunk, cursor, context, self._file, selffile, parent - ) + ) cls = DICTIONARY[fClassName] @@ -2583,9 +2583,13 @@ def get(self): obj, buf = cls.read(buf) uproot_cls = self._file.class_named(fClassName) + if len(uproot_cls.known_versions) > 0: + uproot_cls = uproot_cls.class_of_version(max(uproot_cls.known_versions)) AdapterClass = create_adapter_class(uproot_cls) out = AdapterClass(obj) + out._file = self._file + except uproot.deserialization.DeserializationError: breadcrumbs = context.get("breadcrumbs") From cd7f8ebbcc41ef1fc8986d60992a213bae586484 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Aug 2025 14:39:36 +0000 Subject: [PATCH 10/10] style: pre-commit fixes --- src/uproot/reading.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/uproot/reading.py b/src/uproot/reading.py index 07f1a48e9..8bc70a09e 100644 --- a/src/uproot/reading.py +++ b/src/uproot/reading.py @@ -2568,11 +2568,11 @@ def get(self): try: fClassName = self._fClassName - if fClassName not in DICTIONARY: + if fClassName not in DICTIONARY: cls = self._file.class_named(self._fClassName) return cls.read( chunk, cursor, context, self._file, selffile, parent - ) + ) cls = DICTIONARY[fClassName] @@ -2584,7 +2584,9 @@ def get(self): uproot_cls = self._file.class_named(fClassName) if len(uproot_cls.known_versions) > 0: - uproot_cls = uproot_cls.class_of_version(max(uproot_cls.known_versions)) + uproot_cls = uproot_cls.class_of_version( + max(uproot_cls.known_versions) + ) AdapterClass = create_adapter_class(uproot_cls) out = AdapterClass(obj)