Skip to content

Commit 6537f42

Browse files
authored
Merge pull request #56 from ngoldbaum/missing-data
Initial support for missing data
2 parents 26c270c + 493b258 commit 6537f42

File tree

23 files changed

+275
-113
lines changed

23 files changed

+275
-113
lines changed

.pre-commit-config.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,39 +7,39 @@ repos:
77
language: system
88
require_serial: true
99
entry: |
10-
bash -c 'cd metadatadtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
10+
bash -c 'cd metadatadtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
1111
fail_fast: false
1212
- id: generate-compilation-database-asciidtype
1313
name: Generate compilation database [asciidtype]
1414
files: asciidtype/(meson\.build$|.*\.(c|h)$)
1515
language: system
1616
require_serial: true
1717
entry: |
18-
bash -c 'cd asciidtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
18+
bash -c 'cd asciidtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
1919
fail_fast: false
2020
- id: generate-compilation-database-quaddtype
2121
name: Generate compilation database [quaddtype]
2222
files: quaddtype/(meson\.build$|.*\.(c|h)$)
2323
language: system
2424
require_serial: true
2525
entry: |
26-
bash -c 'cd quaddtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
26+
bash -c 'cd quaddtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
2727
fail_fast: false
2828
- id: generate-compilation-database-unytdtype
2929
name: Generate compilation database [unytdtype]
3030
files: unytdtype/(meson\.build$|.*\.(c|h)$)
3131
language: system
3232
require_serial: true
3333
entry: |
34-
bash -c 'cd unytdtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
34+
bash -c 'cd unytdtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
3535
fail_fast: false
3636
- id: generate-compilation-database-stringdtype
3737
name: Generate compilation database [stringdtype]
3838
files: stringdtype/(meson\.build$|.*\.(c|h)$)
3939
language: system
4040
require_serial: true
4141
entry: |
42-
bash -c 'cd stringdtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
42+
bash -c 'cd stringdtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
4343
fail_fast: false
4444
- repo: https://github.com/pocc/pre-commit-hooks
4545
rev: v1.3.5

asciidtype/asciidtype/src/asciidtype_main.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ PyInit__asciidtype_main(void)
2222
return NULL;
2323
}
2424

25-
if (import_experimental_dtype_api(9) < 0) {
25+
if (import_experimental_dtype_api(10) < 0) {
2626
return NULL;
2727
}
2828

asciidtype/meson.build

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ py.install_sources(
3535
'asciidtype/__init__.py',
3636
'asciidtype/scalar.py'
3737
],
38-
subdir: 'asciidtype'
38+
subdir: 'asciidtype',
39+
pure: false
3940
)
4041

4142
py.extension_module(

metadatadtype/meson.build

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ py.install_sources(
3535
'metadatadtype/__init__.py',
3636
'metadatadtype/scalar.py'
3737
],
38-
subdir: 'metadatadtype'
38+
subdir: 'metadatadtype',
39+
pure: false
3940
)
4041

4142
py.extension_module(

metadatadtype/metadatadtype/src/metadatadtype_main.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ PyInit__metadatadtype_main(void)
2121
if (_import_array() < 0) {
2222
return NULL;
2323
}
24-
if (import_experimental_dtype_api(9) < 0) {
24+
if (import_experimental_dtype_api(10) < 0) {
2525
return NULL;
2626
}
2727

mpfdtype/meson.build

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ py.install_sources(
4646
[
4747
'mpfdtype/__init__.py',
4848
],
49-
subdir: 'mpfdtype'
49+
subdir: 'mpfdtype',
50+
pure: false
5051
)
5152

5253
py.extension_module(

mpfdtype/mpfdtype/src/mpfdtype_main.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ PyInit__mpfdtype_main(void)
2222
if (_import_array() < 0) {
2323
return NULL;
2424
}
25-
if (import_experimental_dtype_api(9) < 0) {
25+
if (import_experimental_dtype_api(10) < 0) {
2626
return NULL;
2727
}
2828

quaddtype/meson.build

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ py.install_sources(
3333
'quaddtype/__init__.py',
3434
'quaddtype/quadscalar.py'
3535
],
36-
subdir: 'quaddtype'
36+
subdir: 'quaddtype',
37+
pure: false
3738
)
3839

3940
py.extension_module(

quaddtype/quaddtype/src/quaddtype_main.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ PyInit__quaddtype_main(void)
2323
return NULL;
2424

2525
// Fail to init if the experimental DType API version 5 isn't supported
26-
if (import_experimental_dtype_api(9) < 0) {
26+
if (import_experimental_dtype_api(10) < 0) {
2727
PyErr_SetString(PyExc_ImportError,
2828
"Error encountered importing the experimental dtype API.");
2929
return NULL;

stringdtype/meson.build

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,11 @@ srcs = [
3535
py.install_sources(
3636
[
3737
'stringdtype/__init__.py',
38-
'stringdtype/scalar.py'
38+
'stringdtype/scalar.py',
39+
'stringdtype/missing.py',
3940
],
40-
subdir: 'stringdtype'
41+
subdir: 'stringdtype',
42+
pure: false
4143
)
4244

4345
py.extension_module(

stringdtype/pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,6 @@ per-file-ignores = {"__init__.py" = ["F401"]}
3131

3232
[tool.meson-python.args]
3333
dist = []
34-
setup = ["-Ddebug=true", "-Doptimization=2"]
34+
setup = ["-Ddebug=true", "-Doptimization=0"]
3535
compile = []
3636
install = []

stringdtype/stringdtype/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@
22
33
"""
44

5+
from .missing import NA # isort: skip
56
from .scalar import StringScalar # isort: skip
67
from ._main import StringDType, _memory_usage
78

89
__all__ = [
10+
"NA",
911
"StringDType",
1012
"StringScalar",
1113
"_memory_usage",

stringdtype/stringdtype/missing.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
class NAType:
2+
def __repr__(self):
3+
return "stringdtype.NA"
4+
5+
6+
NA = NAType()

stringdtype/stringdtype/src/casts.c

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -53,15 +53,13 @@ string_to_string(PyArrayMethod_Context *NPY_UNUSED(context),
5353
npy_intp out_stride = strides[1];
5454

5555
ss *s = NULL;
56+
ss *os = NULL;
5657

5758
while (N--) {
58-
// *out* may be reallocated later; *in->buf* may point to a statically
59-
// allocated empty ss struct, so we need to load the string into an
60-
// intermediate buffer *s* to avoid the possibility of freeing static
61-
// data later on.
62-
load_string(in, (ss **)&s);
63-
ssfree((ss *)out);
64-
if (ssdup((ss *)s, (ss *)out) < 0) {
59+
s = (ss *)in;
60+
os = (ss *)out;
61+
ssfree(os);
62+
if (ssdup(s, os) < 0) {
6563
gil_error(PyExc_MemoryError, "ssdup failed");
6664
return -1;
6765
}
@@ -338,9 +336,18 @@ string_to_unicode(PyArrayMethod_Context *context, char *const data[],
338336
ss *s = NULL;
339337

340338
while (N--) {
341-
load_string(in, &s);
342-
unsigned char *this_string = (unsigned char *)(s->buf);
343-
size_t n_bytes = s->len;
339+
s = (ss *)in;
340+
unsigned char *this_string = NULL;
341+
size_t n_bytes;
342+
if (ss_isnull(s)) {
343+
// lossy but not much else we can do
344+
this_string = (unsigned char *)"NA";
345+
n_bytes = 3;
346+
}
347+
else {
348+
this_string = (unsigned char *)(s->buf);
349+
n_bytes = s->len;
350+
}
344351
size_t tot_n_bytes = 0;
345352

346353
for (int i = 0; i < max_out_size; i++) {
@@ -401,7 +408,7 @@ string_to_bool_resolve_descriptors(PyObject *NPY_UNUSED(self),
401408
}
402409

403410
static int
404-
string_to_bool(PyArrayMethod_Context *context, char *const data[],
411+
string_to_bool(PyArrayMethod_Context *NPY_UNUSED(context), char *const data[],
405412
npy_intp const dimensions[], npy_intp const strides[],
406413
NpyAuxData *NPY_UNUSED(auxdata))
407414
{
@@ -415,8 +422,12 @@ string_to_bool(PyArrayMethod_Context *context, char *const data[],
415422
ss *s = NULL;
416423

417424
while (N--) {
418-
load_string(in, &s);
419-
if (s->len == 0) {
425+
s = (ss *)in;
426+
if (ss_isnull(s)) {
427+
// numpy treats NaN as truthy, following python
428+
*out = (npy_bool)1;
429+
}
430+
else if (s->len == 0) {
420431
*out = (npy_bool)0;
421432
}
422433
else {

0 commit comments

Comments
 (0)