Merge pull request #56 from ngoldbaum/missing-data

Initial support for missing data

Author: ngoldbaum

.pre-commit-config.yaml

@@ -7,39 +7,39 @@ repos:
         language: system
         require_serial: true
         entry: |
-          bash -c 'cd metadatadtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
+          bash -c 'cd metadatadtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
         fail_fast: false
       - id: generate-compilation-database-asciidtype
         name: Generate compilation database [asciidtype]
         files: asciidtype/(meson\.build$|.*\.(c|h)$)
         language: system
         require_serial: true
         entry: |
-          bash -c 'cd asciidtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
+          bash -c 'cd asciidtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
         fail_fast: false
       - id: generate-compilation-database-quaddtype
         name: Generate compilation database [quaddtype]
         files: quaddtype/(meson\.build$|.*\.(c|h)$)
         language: system
         require_serial: true
         entry: |
-          bash -c 'cd quaddtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
+          bash -c 'cd quaddtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
         fail_fast: false
       - id: generate-compilation-database-unytdtype
         name: Generate compilation database [unytdtype]
         files: unytdtype/(meson\.build$|.*\.(c|h)$)
         language: system
         require_serial: true
         entry: |
-          bash -c 'cd unytdtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
+          bash -c 'cd unytdtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
         fail_fast: false
       - id: generate-compilation-database-stringdtype
         name: Generate compilation database [stringdtype]
         files: stringdtype/(meson\.build$|.*\.(c|h)$)
         language: system
         require_serial: true
         entry: |
-          bash -c 'cd stringdtype && mkdir -p build && pip install build meson-python patchelf wheel && python -m build --wheel --no-isolation -Cbuilddir=build';
+          bash -c 'cd stringdtype && mkdir -p build && pip install build meson-python patchelf wheel && meson setup build && python -m build --wheel --no-isolation -Cbuilddir=build';
         fail_fast: false
   - repo: https://github.com/pocc/pre-commit-hooks
     rev: v1.3.5

asciidtype/asciidtype/src/asciidtype_main.c

@@ -22,7 +22,7 @@ PyInit__asciidtype_main(void)
         return NULL;
     }
 
-    if (import_experimental_dtype_api(9) < 0) {
+    if (import_experimental_dtype_api(10) < 0) {
         return NULL;
     }
 

asciidtype/meson.build

@@ -35,7 +35,8 @@ py.install_sources(
     'asciidtype/__init__.py',
     'asciidtype/scalar.py'
   ],
-  subdir: 'asciidtype'
+  subdir: 'asciidtype',
+  pure: false
 )
 
 py.extension_module(

metadatadtype/meson.build

@@ -35,7 +35,8 @@ py.install_sources(
     'metadatadtype/__init__.py',
     'metadatadtype/scalar.py'
   ],
-  subdir: 'metadatadtype'
+  subdir: 'metadatadtype',
+  pure: false
 )
 
 py.extension_module(

metadatadtype/metadatadtype/src/metadatadtype_main.c

@@ -21,7 +21,7 @@ PyInit__metadatadtype_main(void)
     if (_import_array() < 0) {
         return NULL;
     }
-    if (import_experimental_dtype_api(9) < 0) {
+    if (import_experimental_dtype_api(10) < 0) {
         return NULL;
     }
 

mpfdtype/meson.build

@@ -46,7 +46,8 @@ py.install_sources(
   [
     'mpfdtype/__init__.py',
   ],
-  subdir: 'mpfdtype'
+  subdir: 'mpfdtype',
+  pure: false
 )
 
 py.extension_module(

mpfdtype/mpfdtype/src/mpfdtype_main.c

@@ -22,7 +22,7 @@ PyInit__mpfdtype_main(void)
     if (_import_array() < 0) {
         return NULL;
     }
-    if (import_experimental_dtype_api(9) < 0) {
+    if (import_experimental_dtype_api(10) < 0) {
         return NULL;
     }
 

quaddtype/meson.build

@@ -33,7 +33,8 @@ py.install_sources(
     'quaddtype/__init__.py',
     'quaddtype/quadscalar.py'
   ],
-  subdir: 'quaddtype'
+  subdir: 'quaddtype',
+  pure: false
 )
 
 py.extension_module(

quaddtype/quaddtype/src/quaddtype_main.c

@@ -23,7 +23,7 @@ PyInit__quaddtype_main(void)
         return NULL;
 
     // Fail to init if the experimental DType API version 5 isn't supported
-    if (import_experimental_dtype_api(9) < 0) {
+    if (import_experimental_dtype_api(10) < 0) {
         PyErr_SetString(PyExc_ImportError,
                         "Error encountered importing the experimental dtype API.");
         return NULL;

stringdtype/meson.build

@@ -35,9 +35,11 @@ srcs = [
 py.install_sources(
   [
     'stringdtype/__init__.py',
-    'stringdtype/scalar.py'
+    'stringdtype/scalar.py',
+    'stringdtype/missing.py',
   ],
-  subdir: 'stringdtype'
+  subdir: 'stringdtype',
+  pure: false
 )
 
 py.extension_module(

stringdtype/pyproject.toml

@@ -31,6 +31,6 @@ per-file-ignores = {"__init__.py" = ["F401"]}
 
 [tool.meson-python.args]
 dist = []
-setup = ["-Ddebug=true", "-Doptimization=2"]
+setup = ["-Ddebug=true", "-Doptimization=0"]
 compile = []
 install = []

stringdtype/stringdtype/__init__.py

@@ -2,10 +2,12 @@
 
 """
 
+from .missing import NA  # isort: skip
 from .scalar import StringScalar  # isort: skip
 from ._main import StringDType, _memory_usage
 
 __all__ = [
+    "NA",
     "StringDType",
     "StringScalar",
     "_memory_usage",

stringdtype/stringdtype/missing.py

@@ -0,0 +1,6 @@
+class NAType:
+    def __repr__(self):
+        return "stringdtype.NA"
+
+
+NA = NAType()

stringdtype/stringdtype/src/casts.c

@@ -53,15 +53,13 @@ string_to_string(PyArrayMethod_Context *NPY_UNUSED(context),
     npy_intp out_stride = strides[1];
 
     ss *s = NULL;
+    ss *os = NULL;
 
     while (N--) {
-        // *out* may be reallocated later; *in->buf* may point to a statically
-        // allocated empty ss struct, so we need to load the string into an
-        // intermediate buffer *s* to avoid the possibility of freeing static
-        // data later on.
-        load_string(in, (ss **)&s);
-        ssfree((ss *)out);
-        if (ssdup((ss *)s, (ss *)out) < 0) {
+        s = (ss *)in;
+        os = (ss *)out;
+        ssfree(os);
+        if (ssdup(s, os) < 0) {
             gil_error(PyExc_MemoryError, "ssdup failed");
             return -1;
         }
@@ -338,9 +336,18 @@ string_to_unicode(PyArrayMethod_Context *context, char *const data[],
     ss *s = NULL;
 
     while (N--) {
-        load_string(in, &s);
-        unsigned char *this_string = (unsigned char *)(s->buf);
-        size_t n_bytes = s->len;
+        s = (ss *)in;
+        unsigned char *this_string = NULL;
+        size_t n_bytes;
+        if (ss_isnull(s)) {
+            // lossy but not much else we can do
+            this_string = (unsigned char *)"NA";
+            n_bytes = 3;
+        }
+        else {
+            this_string = (unsigned char *)(s->buf);
+            n_bytes = s->len;
+        }
         size_t tot_n_bytes = 0;
 
         for (int i = 0; i < max_out_size; i++) {
@@ -401,7 +408,7 @@ string_to_bool_resolve_descriptors(PyObject *NPY_UNUSED(self),
 }
 
 static int
-string_to_bool(PyArrayMethod_Context *context, char *const data[],
+string_to_bool(PyArrayMethod_Context *NPY_UNUSED(context), char *const data[],
                npy_intp const dimensions[], npy_intp const strides[],
                NpyAuxData *NPY_UNUSED(auxdata))
 {
@@ -415,8 +422,12 @@ string_to_bool(PyArrayMethod_Context *context, char *const data[],
     ss *s = NULL;
 
     while (N--) {
-        load_string(in, &s);
-        if (s->len == 0) {
+        s = (ss *)in;
+        if (ss_isnull(s)) {
+            // numpy treats NaN as truthy, following python
+            *out = (npy_bool)1;
+        }
+        else if (s->len == 0) {
             *out = (npy_bool)0;
         }
         else {