Skip to content

Initial support for missing data #56

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Apr 21, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,39 +7,39 @@ repos:
language: system
require_serial: true
entry: |
bash -c 'cd metadatadtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
bash -c 'cd metadatadtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
fail_fast: false
- id: generate-compilation-database-asciidtype
name: Generate compilation database [asciidtype]
files: asciidtype/(meson\.build$|.*\.(c|h)$)
language: system
require_serial: true
entry: |
bash -c 'cd asciidtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
bash -c 'cd asciidtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
fail_fast: false
- id: generate-compilation-database-quaddtype
name: Generate compilation database [quaddtype]
files: quaddtype/(meson\.build$|.*\.(c|h)$)
language: system
require_serial: true
entry: |
bash -c 'cd quaddtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
bash -c 'cd quaddtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
fail_fast: false
- id: generate-compilation-database-unytdtype
name: Generate compilation database [unytdtype]
files: unytdtype/(meson\.build$|.*\.(c|h)$)
language: system
require_serial: true
entry: |
bash -c 'cd unytdtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
bash -c 'cd unytdtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
fail_fast: false
- id: generate-compilation-database-stringdtype
name: Generate compilation database [stringdtype]
files: stringdtype/(meson\.build$|.*\.(c|h)$)
language: system
require_serial: true
entry: |
bash -c 'cd stringdtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
bash -c 'cd stringdtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
fail_fast: false
- repo: https://github.com/pocc/pre-commit-hooks
rev: v1.3.5
Expand Down
2 changes: 1 addition & 1 deletion asciidtype/asciidtype/src/asciidtype_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ PyInit__asciidtype_main(void)
return NULL;
}

if (import_experimental_dtype_api(9) < 0) {
if (import_experimental_dtype_api(10) < 0) {
return NULL;
}

Expand Down
3 changes: 2 additions & 1 deletion asciidtype/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ py.install_sources(
'asciidtype/__init__.py',
'asciidtype/scalar.py'
],
subdir: 'asciidtype'
subdir: 'asciidtype',
pure: false
)

py.extension_module(
Expand Down
3 changes: 2 additions & 1 deletion metadatadtype/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ py.install_sources(
'metadatadtype/__init__.py',
'metadatadtype/scalar.py'
],
subdir: 'metadatadtype'
subdir: 'metadatadtype',
pure: false
)

py.extension_module(
Expand Down
2 changes: 1 addition & 1 deletion metadatadtype/metadatadtype/src/metadatadtype_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ PyInit__metadatadtype_main(void)
if (_import_array() < 0) {
return NULL;
}
if (import_experimental_dtype_api(9) < 0) {
if (import_experimental_dtype_api(10) < 0) {
return NULL;
}

Expand Down
3 changes: 2 additions & 1 deletion mpfdtype/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ py.install_sources(
[
'mpfdtype/__init__.py',
],
subdir: 'mpfdtype'
subdir: 'mpfdtype',
pure: false
)

py.extension_module(
Expand Down
2 changes: 1 addition & 1 deletion mpfdtype/mpfdtype/src/mpfdtype_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ PyInit__mpfdtype_main(void)
if (_import_array() < 0) {
return NULL;
}
if (import_experimental_dtype_api(9) < 0) {
if (import_experimental_dtype_api(10) < 0) {
return NULL;
}

Expand Down
3 changes: 2 additions & 1 deletion quaddtype/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ py.install_sources(
'quaddtype/__init__.py',
'quaddtype/quadscalar.py'
],
subdir: 'quaddtype'
subdir: 'quaddtype',
pure: false
)

py.extension_module(
Expand Down
2 changes: 1 addition & 1 deletion quaddtype/quaddtype/src/quaddtype_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ PyInit__quaddtype_main(void)
return NULL;

// Fail to init if the experimental DType API version 5 isn't supported
if (import_experimental_dtype_api(9) < 0) {
if (import_experimental_dtype_api(10) < 0) {
PyErr_SetString(PyExc_ImportError,
"Error encountered importing the experimental dtype API.");
return NULL;
Expand Down
6 changes: 4 additions & 2 deletions stringdtype/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,11 @@ srcs = [
py.install_sources(
[
'stringdtype/__init__.py',
'stringdtype/scalar.py'
'stringdtype/scalar.py',
'stringdtype/missing.py',
],
subdir: 'stringdtype'
subdir: 'stringdtype',
pure: false
)

py.extension_module(
Expand Down
2 changes: 1 addition & 1 deletion stringdtype/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,6 @@ per-file-ignores = {"__init__.py" = ["F401"]}

[tool.meson-python.args]
dist = []
setup = ["-Ddebug=true", "-Doptimization=2"]
setup = ["-Ddebug=true", "-Doptimization=0"]
compile = []
install = []
2 changes: 2 additions & 0 deletions stringdtype/stringdtype/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@

"""

from .missing import NA # isort: skip
from .scalar import StringScalar # isort: skip
from ._main import StringDType, _memory_usage

__all__ = [
"NA",
"StringDType",
"StringScalar",
"_memory_usage",
Expand Down
6 changes: 6 additions & 0 deletions stringdtype/stringdtype/missing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
class NAType:
def __repr__(self):
return "stringdtype.NA"


NA = NAType()
37 changes: 24 additions & 13 deletions stringdtype/stringdtype/src/casts.c
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,13 @@ string_to_string(PyArrayMethod_Context *NPY_UNUSED(context),
npy_intp out_stride = strides[1];

ss *s = NULL;
ss *os = NULL;

while (N--) {
// *out* may be reallocated later; *in->buf* may point to a statically
// allocated empty ss struct, so we need to load the string into an
// intermediate buffer *s* to avoid the possibility of freeing static
// data later on.
load_string(in, (ss **)&s);
ssfree((ss *)out);
if (ssdup((ss *)s, (ss *)out) < 0) {
s = (ss *)in;
os = (ss *)out;
ssfree(os);
if (ssdup(s, os) < 0) {
gil_error(PyExc_MemoryError, "ssdup failed");
return -1;
}
Expand Down Expand Up @@ -338,9 +336,18 @@ string_to_unicode(PyArrayMethod_Context *context, char *const data[],
ss *s = NULL;

while (N--) {
load_string(in, &s);
unsigned char *this_string = (unsigned char *)(s->buf);
size_t n_bytes = s->len;
s = (ss *)in;
unsigned char *this_string = NULL;
size_t n_bytes;
if (ss_isnull(s)) {
// lossy but not much else we can do
this_string = (unsigned char *)"NA";
n_bytes = 3;
}
else {
this_string = (unsigned char *)(s->buf);
n_bytes = s->len;
}
size_t tot_n_bytes = 0;

for (int i = 0; i < max_out_size; i++) {
Expand Down Expand Up @@ -401,7 +408,7 @@ string_to_bool_resolve_descriptors(PyObject *NPY_UNUSED(self),
}

static int
string_to_bool(PyArrayMethod_Context *context, char *const data[],
string_to_bool(PyArrayMethod_Context *NPY_UNUSED(context), char *const data[],
npy_intp const dimensions[], npy_intp const strides[],
NpyAuxData *NPY_UNUSED(auxdata))
{
Expand All @@ -415,8 +422,12 @@ string_to_bool(PyArrayMethod_Context *context, char *const data[],
ss *s = NULL;

while (N--) {
load_string(in, &s);
if (s->len == 0) {
s = (ss *)in;
if (ss_isnull(s)) {
// numpy treats NaN as truthy, following python
*out = (npy_bool)1;
}
else if (s->len == 0) {
*out = (npy_bool)0;
}
else {
Expand Down
Loading