Skip to content

[Bug]: Cannot build data (dataset) with dtype datetime #1311

@rly

Description

@rly

What happened?

See nehatk17/ndx-multisubjects#4

Steps to Reproduce

Create datasetspec that extends VectorData with dtype: datetime

Traceback

../hdmf/src/hdmf/utils.py:578: in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/backends/hdf5/h5tools.py:353: in write
    super().write(**kwargs)
../hdmf/src/hdmf/utils.py:578: in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/backends/io.py:98: in write
    f_builder = self.__manager.build(container, source=self.__source, root=True)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/utils.py:578: in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/build/manager.py:173: in build
    result = self.__type_map.build(container, self, source=source, spec_ext=spec_ext, export=export)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/utils.py:578: in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/build/manager.py:789: in build
    builder = obj_mapper.build(container, manager, builder=builder, source=source, spec_ext=spec_ext, export=export)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/utils.py:578: in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/build/objectmapper.py:765: in build
    self.__add_groups(builder, self.__spec.groups, container, manager, source, export)
../hdmf/src/hdmf/build/objectmapper.py:1164: in __add_groups
    self.__add_groups(sub_builder, spec.groups, container, build_manager, source, export)
../hdmf/src/hdmf/build/objectmapper.py:1177: in __add_groups
    self.__add_containers(builder, spec, attr_value, build_manager, source, container, export)
../hdmf/src/hdmf/build/objectmapper.py:1197: in __add_containers
    new_builder = build_manager.build(value, source=source, spec_ext=spec, export=export)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/utils.py:578: in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/build/manager.py:173: in build
    result = self.__type_map.build(container, self, source=source, spec_ext=spec_ext, export=export)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/utils.py:578: in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/build/manager.py:789: in build
    builder = obj_mapper.build(container, manager, builder=builder, source=source, spec_ext=spec_ext, export=export)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/utils.py:578: in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/build/objectmapper.py:764: in build
    self.__add_datasets(builder, self.__spec.datasets, container, manager, source, export)
../hdmf/src/hdmf/build/objectmapper.py:1147: in __add_datasets
    self.__add_containers(builder, spec, attr_value, build_manager, source, container, export)
../hdmf/src/hdmf/build/objectmapper.py:1197: in __add_containers
    new_builder = build_manager.build(value, source=source, spec_ext=spec, export=export)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/utils.py:578: in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/build/manager.py:173: in build
    result = self.__type_map.build(container, self, source=source, spec_ext=spec_ext, export=export)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/utils.py:578: in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/build/manager.py:789: in build
    builder = obj_mapper.build(container, manager, builder=builder, source=source, spec_ext=spec_ext, export=export)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../hdmf/src/hdmf/utils.py:578: in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
           ^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <pynwb.io.core.VectorDataMap object at 0x10e8be850>
kwargs = {'builder': None, 'container': <hdmf.common.table.VectorData object at 0x10f88caa0>, 'export': False, 'manager': <hdmf.build.manager.BuildManager object at 0x10e8be350>, ...}
container = <hdmf.common.table.VectorData object at 0x10f88caa0>, manager = <hdmf.build.manager.BuildManager object at 0x10e8be350>, parent = None
source = '/Users/rly/Documents/NWB/ndx-multisubjects/test.nwb', builder = None
spec_ext = {'dtype': 'isodatetime', 'doc': 'Date of birth of subject. Can be supplied instead of age.', 'name': 'date_of_birth', 'quantity': '?', 'neurodata_type_inc': 'VectorData'}
export = False

    @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"},
            {"name": "manager", "type": BuildManager, "doc": "the BuildManager to use for managing this build"},
            {"name": "parent", "type": GroupBuilder, "doc": "the parent of the resulting Builder", 'default': None},
            {"name": "source", "type": str,
             "doc": "the source of container being built i.e. file path", 'default': None},
            {"name": "builder", "type": BaseBuilder, "doc": "the Builder to build on", 'default': None},
            {"name": "spec_ext", "type": BaseStorageSpec, "doc": "a spec extension", 'default': None},
            {"name": "export", "type": bool, "doc": "whether this build is for exporting",
             'default': False},
            returns="the Builder representing the given AbstractContainer", rtype=Builder)
    def build(self, **kwargs):
        '''Convert an AbstractContainer to a Builder representation.
    
        References are not added but are queued to be added in the BuildManager.
        '''
        container, manager, parent, source = getargs('container', 'manager', 'parent', 'source', kwargs)
        builder, spec_ext, export = getargs('builder', 'spec_ext', 'export', kwargs)
        name = manager.get_builder_name(container)
        if isinstance(self.__spec, GroupSpec):
            self.logger.debug("Building %s '%s' as a group (source: %s)"
                              % (container.__class__.__name__, container.name, repr(source)))
            if builder is None:
                builder = GroupBuilder(name, parent=parent, source=source)
            self.__add_datasets(builder, self.__spec.datasets, container, manager, source, export)
            self.__add_groups(builder, self.__spec.groups, container, manager, source, export)
            self.__add_links(builder, self.__spec.links, container, manager, source, export)
        else:
            if builder is None:
                if not isinstance(container, Data):
                    msg = "'container' must be of type Data with DatasetSpec"
                    raise ValueError(msg)
                spec_dtype, spec_shape, spec_dims, spec = self.__check_dset_spec(self.spec, spec_ext)
                dimension_labels = self.__get_dimension_labels_from_spec(container.data, spec_shape, spec_dims)
                if isinstance(spec_dtype, RefSpec):
                    self.logger.debug("Building %s '%s' as a dataset of references (source: %s)"
                                      % (container.__class__.__name__, container.name, repr(source)))
                    # create dataset builder with data=None as a placeholder. fill in with refs later
                    builder = DatasetBuilder(
                        name,
                        data=None,
                        parent=parent,
                        source=source,
                        dtype=spec_dtype.reftype,
                        dimension_labels=dimension_labels,
                    )
                    manager.queue_ref(self.__set_dataset_to_refs(builder, spec_dtype, spec_shape, container, manager))
                elif isinstance(spec_dtype, list):
                    # a compound dataset
                    self.logger.debug("Building %s '%s' as a dataset of compound dtypes (source: %s)"
                                      % (container.__class__.__name__, container.name, repr(source)))
                    # create dataset builder with data=None, dtype=None as a placeholder. fill in with refs later
                    builder = DatasetBuilder(
                        name,
                        data=None,
                        parent=parent,
                        source=source,
                        dtype=spec_dtype,
                        dimension_labels=dimension_labels,
                    )
                    manager.queue_ref(self.__set_compound_dataset_to_refs(builder, spec, spec_dtype, container,
                                                                          manager))
                else:
                    # a regular dtype
                    if spec_dtype is None and self.__is_reftype(container.data):
                        self.logger.debug("Building %s '%s' containing references as a dataset of unspecified dtype "
                                          "(source: %s)"
                                          % (container.__class__.__name__, container.name, repr(source)))
                        # an unspecified dtype and we were given references
                        # create dataset builder with data=None as a placeholder. fill in with refs later
                        builder = DatasetBuilder(
                            name,
                            data=None,
                            parent=parent,
                            source=source,
                            dtype="object",
                            dimension_labels=dimension_labels,
                        )
                        manager.queue_ref(self.__set_untyped_dataset_to_refs(builder, container, manager))
                    else:
                        # a dataset that has no references, pass the conversion off to the convert_dtype method
                        self.logger.debug("Building %s '%s' as a dataset (source: %s)"
                                          % (container.__class__.__name__, container.name, repr(source)))
                        try:
                            # use spec_dtype from self.spec when spec_ext does not specify dtype
                            if isinstance(container.data, TermSetWrapper):
                                data = container.data.value
                            else:
                                data = container.data
                            bldr_data, dtype = self.convert_dtype(spec, data, spec_dtype=spec_dtype)
                        except Exception as ex:
                            msg = f"could not resolve dtype for {type(container).__name__} '{container.name}'"
                            full_msg = f"{msg}: {str(ex)}"
>                           raise Exception(full_msg) from ex
E                           Exception: could not resolve dtype for VectorData 'date_of_birth': Expected unicode or ascii string, got <class 'datetime.date'>

../hdmf/src/hdmf/build/objectmapper.py:833: Exception

Operating System

macOS

Python Version

3.13

Package Versions

latest dev version of hdmf

Metadata

Metadata

Assignees

Labels

category: bugerrors in the code or code behaviorpriority: mediumnon-critical problem and/or affecting only a small set of users

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions