From a83850d83f62f5c1471d931e505007008f7e09bc Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 22 Dec 2021 12:39:19 +0100 Subject: [PATCH 1/5] add prototype dataset --- .../prototype/datasets/_builtin/__init__.py | 1 + .../prototype/datasets/_builtin/clevr.py | 121 ++++++++++++++++++ .../prototype/datasets/utils/_internal.py | 10 +- 3 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 torchvision/prototype/datasets/_builtin/clevr.py diff --git a/torchvision/prototype/datasets/_builtin/__init__.py b/torchvision/prototype/datasets/_builtin/__init__.py index 62abc3119f6..93628b794ad 100644 --- a/torchvision/prototype/datasets/_builtin/__init__.py +++ b/torchvision/prototype/datasets/_builtin/__init__.py @@ -1,6 +1,7 @@ from .caltech import Caltech101, Caltech256 from .celeba import CelebA from .cifar import Cifar10, Cifar100 +from .clevr import CLEVR from .coco import Coco from .imagenet import ImageNet from .mnist import MNIST, FashionMNIST, KMNIST, EMNIST, QMNIST diff --git a/torchvision/prototype/datasets/_builtin/clevr.py b/torchvision/prototype/datasets/_builtin/clevr.py new file mode 100644 index 00000000000..5b6bd787fd3 --- /dev/null +++ b/torchvision/prototype/datasets/_builtin/clevr.py @@ -0,0 +1,121 @@ +import functools +import io +import pathlib +import re +from typing import Any, Callable, Dict, List, Optional, Tuple + +import torch +from torchdata.datapipes.iter import IterDataPipe, Mapper, Filter, IterKeyZipper, Demultiplexer, JsonParser, UnBatcher +from torchvision.prototype.datasets.utils import ( + Dataset, + DatasetConfig, + DatasetInfo, + HttpResource, + OnlineResource, + DatasetType, +) +from torchvision.prototype.datasets.utils._internal import ( + INFINITE_BUFFER_SIZE, + hint_sharding, + hint_shuffling, + path_comparator, + MappingIterator, + path_accessor, + getitem, +) +from torchvision.prototype.features import Label + + +class CLEVR(Dataset): + def _make_info(self) -> DatasetInfo: + return DatasetInfo( + "clevr", + type=DatasetType.IMAGE, + homepage="https://cs.stanford.edu/people/jcjohns/clevr/", + valid_options=dict(split=("train", "val", "test")), + ) + + def resources(self, config: DatasetConfig) -> List[OnlineResource]: + archive = HttpResource( + "https://dl.fbaipublicfiles.com/clevr/CLEVR_v1.0.zip", + sha256="5cd61cf1096ed20944df93c9adb31e74d189b8459a94f54ba00090e5c59936d1", + ) + return [archive] + + def _classify_archive(self, data: Tuple[str, Any]) -> Optional[int]: + path = pathlib.Path(data[0]) + if path.parents[1].name == "images": + return 0 + elif path.parent.name == "scenes": + return 1 + else: + return None + + _ANNS_NAME_PATTERN = re.compile(r"CLEVR_(?Ptrain|val)_scenes[.]json") + + def _filter_scene_files(self, data: Tuple[str, Any], *, split: str) -> bool: + path = pathlib.Path(data[0]) + return self._ANNS_NAME_PATTERN.match(path.name)["split"] == split # type: ignore[index] + + def _filter_scene_anns(self, data: Tuple[str, Any]) -> bool: + key, _ = data + return key == "scenes" + + def _add_empty_anns(self, data: Tuple[str, io.IOBase]) -> Tuple[Tuple[str, io.IOBase], None]: + return data, None + + def _collate_and_decode_sample( + self, + data: Tuple[Tuple[str, io.IOBase], Optional[Dict[str, Any]]], + *, + decoder: Optional[Callable[[io.IOBase], torch.Tensor]], + ) -> Dict[str, Any]: + image_data, scenes_data = data + path, buffer = image_data + + return dict( + path=path, + image=decoder(buffer) if decoder else buffer, + label=Label(len(scenes_data["objects"])) if scenes_data else None, + ) + + def _make_datapipe( + self, + resource_dps: List[IterDataPipe], + *, + config: DatasetConfig, + decoder: Optional[Callable[[io.IOBase], torch.Tensor]], + ) -> IterDataPipe[Dict[str, Any]]: + archive_dp = resource_dps[0] + images_dp, scenes_dp = Demultiplexer( + archive_dp, + 2, + self._classify_archive, + drop_none=True, + buffer_size=INFINITE_BUFFER_SIZE, + ) + + images_dp = Filter(images_dp, path_comparator("parent.name", config.split)) + images_dp = hint_sharding(images_dp) + images_dp = hint_shuffling(images_dp) + + if config.split != "test": + scenes_dp = Filter(scenes_dp, functools.partial(self._filter_scene_files, split=config.split)) + scenes_dp = JsonParser(scenes_dp) + scenes_dp = Mapper(scenes_dp, getitem(1)) + scenes_dp = MappingIterator(scenes_dp) + scenes_dp = Filter(scenes_dp, self._filter_scene_anns) + scenes_dp = Mapper(scenes_dp, getitem(1)) + scenes_dp = UnBatcher(scenes_dp) + + dp = IterKeyZipper( + images_dp, + scenes_dp, + key_fn=path_accessor("name"), + ref_key_fn=getitem("image_filename"), + buffer_size=INFINITE_BUFFER_SIZE, + ) + else: + dp = Mapper(images_dp, self._add_empty_anns) + + return Mapper(dp, functools.partial(self._collate_and_decode_sample, decoder=decoder)) diff --git a/torchvision/prototype/datasets/utils/_internal.py b/torchvision/prototype/datasets/utils/_internal.py index 824594dd28e..0ef14f21441 100644 --- a/torchvision/prototype/datasets/utils/_internal.py +++ b/torchvision/prototype/datasets/utils/_internal.py @@ -108,7 +108,7 @@ def __iter__(self) -> Iterator[Tuple[int, D]]: yield from enumerate(self.datapipe, self.start) -def _getitem_closure(obj: Any, *, items: Tuple[Any, ...]) -> Any: +def _getitem_closure(obj: Any, *, items: Sequence[Any]) -> Any: for item in items: obj = obj[item] return obj @@ -118,8 +118,14 @@ def getitem(*items: Any) -> Callable[[Any], Any]: return functools.partial(_getitem_closure, items=items) +def _getattr_closure(obj: Any, *, attrs: Sequence[str]) -> Any: + for attr in attrs: + obj = getattr(obj, attr) + return obj + + def _path_attribute_accessor(path: pathlib.Path, *, name: str) -> D: - return cast(D, getattr(path, name)) + return cast(D, _getattr_closure(path, attrs=name.split("."))) def _path_accessor_closure(data: Tuple[str, Any], *, getter: Callable[[pathlib.Path], D]) -> D: From 2f178e6a8e3f931158014243cdfe09fb1079c538 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Dec 2021 09:52:03 +0100 Subject: [PATCH 2/5] add old-style dataset --- test/test_datasets.py | 32 ++++++++++++ torchvision/datasets/__init__.py | 2 + torchvision/datasets/clevr.py | 87 ++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+) create mode 100644 torchvision/datasets/clevr.py diff --git a/test/test_datasets.py b/test/test_datasets.py index 761f11d77dc..f8fd22e8eb0 100644 --- a/test/test_datasets.py +++ b/test/test_datasets.py @@ -2168,5 +2168,37 @@ def inject_fake_data(self, tmpdir, config): return num_sequences * (num_examples_per_sequence - 1) +class CLEVRClassificationTestCase(datasets_utils.ImageDatasetTestCase): + DATASET_CLASS = datasets.CLEVRClassification + FEATURE_TYPES = (PIL.Image.Image, (int, type(None))) + + ADDITIONAL_CONFIGS = datasets_utils.combinations_grid(split=("train", "val", "test")) + + def inject_fake_data(self, tmpdir, config): + data_folder = pathlib.Path(tmpdir) / "clevr" / "CLEVR_v1.0" + + images_folder = data_folder / "images" + image_files = datasets_utils.create_image_folder( + images_folder, config["split"], lambda idx: f"CLEVR_{config['split']}_{idx:06d}.png", num_examples=5 + ) + + scenes_folder = data_folder / "scenes" + scenes_folder.mkdir() + if config["split"] != "test": + with open(scenes_folder / f"CLEVR_{config['split']}_scenes.json", "w") as file: + json.dump( + dict( + info=dict(), + scenes=[ + dict(image_filename=image_file.name, objects=[dict()] * int(torch.randint(10, ()))) + for image_file in image_files + ], + ), + file, + ) + + return len(image_files) + + if __name__ == "__main__": unittest.main() diff --git a/torchvision/datasets/__init__.py b/torchvision/datasets/__init__.py index 80859791004..5a7fa7e138b 100644 --- a/torchvision/datasets/__init__.py +++ b/torchvision/datasets/__init__.py @@ -3,6 +3,7 @@ from .celeba import CelebA from .cifar import CIFAR10, CIFAR100 from .cityscapes import Cityscapes +from .clevr import CLEVRClassification from .coco import CocoCaptions, CocoDetection from .fakedata import FakeData from .flickr import Flickr8k, Flickr30k @@ -77,4 +78,5 @@ "FlyingChairs", "FlyingThings3D", "HD1K", + "CLEVRClassification", ) diff --git a/torchvision/datasets/clevr.py b/torchvision/datasets/clevr.py new file mode 100644 index 00000000000..fcd088263fe --- /dev/null +++ b/torchvision/datasets/clevr.py @@ -0,0 +1,87 @@ +import json +import pathlib +from typing import Any, Callable, Optional, Tuple +from urllib.parse import urlparse + +from PIL import Image + +from .utils import download_and_extract_archive, verify_str_arg +from .vision import VisionDataset + + +class CLEVRClassification(VisionDataset): + """`CLEVR `_ classification dataset. + + The number of objects in a scene are used as label. + + Args: + root (string): Root directory of dataset where directory ``clevr`` exists or will be saved to if download is + set to True. + split (string, optional): The dataset split, supports ``"train"`` (default), ``"val"``, or ``"test"``. + transform (callable, optional): A function/transform that takes in an PIL image and returns a transformed + version. E.g, ``transforms.RandomCrop`` + target_transform (callable, optional): A function/transform that takes in them target and transforms it. + download (bool, optional): If true, downloads the dataset from the internet and puts it in root directory. If + dataset is already downloaded, it is not downloaded again. + """ + + _URL = "https://dl.fbaipublicfiles.com/clevr/CLEVR_v1.0.zip" + _MD5 = "b11922020e72d0cd9154779b2d3d07d2" + + def __init__( + self, + root: str, + split: str = "train", + transform: Optional[Callable] = None, + target_transform: Optional[Callable] = None, + download: bool = True, + ) -> None: + self._split = verify_str_arg(split, "split", ("train", "val", "test")) + super().__init__(root, transform=transform, target_transform=target_transform) + self._base_folder = pathlib.Path(self.root) / "clevr" + self._data_folder = self._base_folder / pathlib.Path(urlparse(self._URL).path).stem + + if download: + self._download() + + if not self._check_exists(): + raise RuntimeError("Dataset not found or corrupted. You can use download=True to download it") + + self._image_files = sorted(self._data_folder.joinpath("images", self._split).glob("*")) + + if self._split != "test": + with open(self._data_folder / "scenes" / f"CLEVR_{self._split}_scenes.json") as file: + content = json.load(file) + num_objects = {scene["image_filename"]: len(scene["objects"]) for scene in content["scenes"]} + self._labels = [num_objects[image_file.name] for image_file in self._image_files] + else: + self._labels = [None] * len(self._image_files) + + def __len__(self) -> int: + return len(self._image_files) + + def __getitem__(self, idx: int) -> Tuple[Any, Any]: + image_file = self._image_files[idx] + label = self._labels[idx] + + image = Image.open(image_file).convert("RGB") + + if self.transform: + image = self.transform(image) + + if self.target_transform: + label = self.target_transform(label) + + return image, label + + def _check_exists(self) -> bool: + return self._data_folder.exists() and self._data_folder.is_dir() + + def _download(self) -> None: + if self._check_exists(): + return + + download_and_extract_archive(self._URL, str(self._base_folder), md5=self._MD5) + + def extra_repr(self) -> str: + return f"split={self._split}" From 68a762eb7309dcaaba41d4f2871b03f97b3a1084 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Dec 2021 10:49:39 +0100 Subject: [PATCH 3/5] appease mypy --- torchvision/datasets/clevr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torchvision/datasets/clevr.py b/torchvision/datasets/clevr.py index fcd088263fe..a5e8ff26e02 100644 --- a/torchvision/datasets/clevr.py +++ b/torchvision/datasets/clevr.py @@ -1,6 +1,6 @@ import json import pathlib -from typing import Any, Callable, Optional, Tuple +from typing import Any, Callable, Optional, Tuple, List from urllib.parse import urlparse from PIL import Image @@ -49,6 +49,7 @@ def __init__( self._image_files = sorted(self._data_folder.joinpath("images", self._split).glob("*")) + self._labels: List[Optional[int]] if self._split != "test": with open(self._data_folder / "scenes" / f"CLEVR_{self._split}_scenes.json") as file: content = json.load(file) From 739ac291f2a980b6556b4827a2cae767e6aba27d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 5 Jan 2022 16:09:50 +0100 Subject: [PATCH 4/5] simplify prototype scenes --- torchvision/prototype/datasets/_builtin/clevr.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/clevr.py b/torchvision/prototype/datasets/_builtin/clevr.py index 5b6bd787fd3..447c1b5190d 100644 --- a/torchvision/prototype/datasets/_builtin/clevr.py +++ b/torchvision/prototype/datasets/_builtin/clevr.py @@ -1,7 +1,6 @@ import functools import io import pathlib -import re from typing import Any, Callable, Dict, List, Optional, Tuple import torch @@ -19,7 +18,6 @@ hint_sharding, hint_shuffling, path_comparator, - MappingIterator, path_accessor, getitem, ) @@ -51,12 +49,6 @@ def _classify_archive(self, data: Tuple[str, Any]) -> Optional[int]: else: return None - _ANNS_NAME_PATTERN = re.compile(r"CLEVR_(?Ptrain|val)_scenes[.]json") - - def _filter_scene_files(self, data: Tuple[str, Any], *, split: str) -> bool: - path = pathlib.Path(data[0]) - return self._ANNS_NAME_PATTERN.match(path.name)["split"] == split # type: ignore[index] - def _filter_scene_anns(self, data: Tuple[str, Any]) -> bool: key, _ = data return key == "scenes" @@ -100,12 +92,9 @@ def _make_datapipe( images_dp = hint_shuffling(images_dp) if config.split != "test": - scenes_dp = Filter(scenes_dp, functools.partial(self._filter_scene_files, split=config.split)) + scenes_dp = Filter(scenes_dp, path_comparator("name", f"CLEVR_{config.split}_scenes.json")) scenes_dp = JsonParser(scenes_dp) - scenes_dp = Mapper(scenes_dp, getitem(1)) - scenes_dp = MappingIterator(scenes_dp) - scenes_dp = Filter(scenes_dp, self._filter_scene_anns) - scenes_dp = Mapper(scenes_dp, getitem(1)) + scenes_dp = Mapper(scenes_dp, getitem(1, "scenes")) scenes_dp = UnBatcher(scenes_dp) dp = IterKeyZipper( From ac97d11d64b65627f70db7b96f5fe3b819f9d1c6 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 5 Jan 2022 16:10:12 +0100 Subject: [PATCH 5/5] Update torchvision/datasets/clevr.py Co-authored-by: Nicolas Hug --- torchvision/datasets/clevr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchvision/datasets/clevr.py b/torchvision/datasets/clevr.py index a5e8ff26e02..7ba5ca6cc47 100644 --- a/torchvision/datasets/clevr.py +++ b/torchvision/datasets/clevr.py @@ -15,7 +15,7 @@ class CLEVRClassification(VisionDataset): The number of objects in a scene are used as label. Args: - root (string): Root directory of dataset where directory ``clevr`` exists or will be saved to if download is + root (string): Root directory of dataset where directory ``root/clevr`` exists or will be saved to if download is set to True. split (string, optional): The dataset split, supports ``"train"`` (default), ``"val"``, or ``"test"``. transform (callable, optional): A function/transform that takes in an PIL image and returns a transformed