ENH: GH2578, allow ix and friends to partially set when the key is not contained

      in the object

Author: jreback

doc/source/indexing.rst

@@ -440,6 +440,40 @@ the object it modified, which in the case of enlargement, will be a **new object
    df.at[dates[5], 'E'] = 7
    df.iat[3, 0] = 7
 
+.. _indexing.basics.partial_setting:
+
+Setting With Enlargement
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``.loc/.iloc/[]`` operations can perform enlargement when setting a non-existant key for that axis.
+
+In the ``Series`` case this is effectively an appending operation
+
+.. ipython:: python
+
+   se = Series([1,2,3])
+   se
+   se[5] = 5.
+   se
+
+A ``DataFrame`` can be enlarged on either axis via ``.loc``
+
+.. ipython:: python
+
+   dfi = DataFrame(np.arange(6).reshape(3,2),
+                   columns=['A','B'])
+   dfi
+   dfi.loc[:,'C'] = dfi.loc[:,'A']
+   dfi
+
+This is like an ``append`` operation on the ``DataFrame``.
+
+.. ipython:: python
+
+   dfi.loc[3] = 5
+   dfi
+
+
 Boolean indexing
 ~~~~~~~~~~~~~~~~
 

doc/source/release.rst

@@ -123,6 +123,8 @@ pandas 0.13
 
     - added ``date_unit`` parameter to specify resolution of timestamps. Options
       are seconds, milliseconds, microseconds and nanoseconds. (:issue:`4362`, :issue:`4498`).
+    - allow ``ix/loc/iloc`` for Series/DataFrame/Panel to set on any axis even when the single-key is not currently contained in
+      the index for that axis (:issue:`2578`)
 
   - ``Index`` and ``MultiIndex`` changes (:issue:`4039`):
 
@@ -296,7 +298,7 @@ See :ref:`Internal Refactoring<whatsnew_0130.refactoring>`
   - ``tslib.get_period_field()`` and ``tslib.get_period_field_arr()`` now raise
     if code argument out of range (:issue:`4519`, :issue:`4520`)
   - Fix boolean indexing on an empty series loses index names (:issue:`4235`),
-  infer_dtype works with empty arrays.
+    infer_dtype works with empty arrays.
   - Fix reindexing with multiple axes; if an axes match was not replacing the current axes, leading
     to a possible lazay frequency inference issue (:issue:`3317`)
   - Fixed issue where ``DataFrame.apply`` was reraising exceptions incorrectly

doc/source/v0.13.0.txt

@@ -134,6 +134,54 @@ API changes
         df1 and df2
         s1 and s2
 
+Indexing API Changes
+~~~~~~~~~~~~~~~~~~~~
+
+    Prior to 0.13, it was impossible to use an indexer (``.loc/.iloc/.ix``) to set a value that
+    was not contained in the index of a particular axis. (:issue:`2578`). See more at :ref:`here<indexing.basics.partial_setting>`
+
+    In the ``Series`` case this is effectively an appending operation
+
+    .. ipython:: python
+
+       s = Series([1,2,3])
+       s
+       s[5] = 5.
+       s
+
+    .. ipython:: python
+
+       dfi = DataFrame(np.arange(6).reshape(3,2),
+                       columns=['A','B'])
+       dfi
+
+    This would previously ``KeyError``
+
+    .. ipython:: python
+
+       dfi.loc[:,'C'] = dfi.loc[:,'A']
+       dfi
+
+    This is like an ``append`` operation.
+
+    .. ipython:: python
+
+       dfi.loc[3] = 5
+       dfi
+
+    A Panel setting operation on an arbitrary axis aligns the input to the Panel
+
+    .. ipython:: python
+
+       p = pd.Panel(np.arange(16).reshape(2,4,2),
+                   items=['Item1','Item2'],
+                   major_axis=pd.date_range('2001/1/12',periods=4),
+                   minor_axis=['A','B'],dtype='float64')
+       p
+       p.loc[:,:,'C'] = Series([30,32],index=p.items)
+       p
+       p.loc[:,:,'C']
+
 Enhancements
 ~~~~~~~~~~~~
 

pandas/core/indexing.py

@@ -1,7 +1,7 @@
 # pylint: disable=W0223
 
 from datetime import datetime
-from pandas.core.common import _asarray_tuplesafe
+from pandas.core.common import _asarray_tuplesafe, is_list_like
 from pandas.core.index import Index, MultiIndex, _ensure_index
 from pandas.compat import range, zip
 import pandas.compat as compat
@@ -86,27 +86,66 @@ def __setitem__(self, key, value):
             if len(key) > self.ndim:
                 raise IndexingError('only tuples of length <= %d supported',
                                     self.ndim)
-            indexer = self._convert_tuple(key)
+            indexer = self._convert_tuple(key, is_setter=True)
         else:
-            indexer = self._convert_to_indexer(key)
+            indexer = self._convert_to_indexer(key, is_setter=True)
 
         self._setitem_with_indexer(indexer, value)
 
     def _has_valid_tuple(self, key):
         pass
 
-    def _convert_tuple(self, key):
+    def _convert_tuple(self, key, is_setter=False):
         keyidx = []
         for i, k in enumerate(key):
-            idx = self._convert_to_indexer(k, axis=i)
+            idx = self._convert_to_indexer(k, axis=i, is_setter=is_setter)
             keyidx.append(idx)
         return tuple(keyidx)
 
     def _setitem_with_indexer(self, indexer, value):
 
         # also has the side effect of consolidating in-place
-        # mmm, spaghetti
+        from pandas import Panel, DataFrame, Series
 
+        # maybe partial set
+        if isinstance(indexer,tuple):
+            nindexer = []
+            for i, idx in enumerate(indexer):
+                if isinstance(idx, dict):
+
+                    # reindex the axis to the new value
+                    # and set inplace
+                    key,_ = _convert_missing_indexer(idx)
+                    labels = self.obj._get_axis(i) + Index([key])
+                    self.obj._data = self.obj.reindex_axis(labels,i)._data
+
+                    nindexer.append(labels.get_loc(key))
+                else:
+                    nindexer.append(idx)
+
+            indexer = tuple(nindexer)
+        else:
+
+            indexer, missing = _convert_missing_indexer(indexer)
+
+            if missing:
+
+                # reindex the axis to the new value
+                # and set inplace
+                if self.ndim == 1:
+                    self.obj._data = self.obj.append(Series(value,index=[indexer]))._data
+                    return
+
+                elif self.ndim == 2:
+                    labels = self.obj._get_axis(0) + Index([indexer])
+                    self.obj._data = self.obj.reindex_axis(labels,0)._data
+                    return getattr(self.obj,self.name).__setitem__(indexer,value)
+
+                # set using setitem (Panel and > dims)
+                elif self.ndim >= 3:
+                    return self.obj.__setitem__(indexer,value)
+
+        # align and set the values
         if self.obj._is_mixed_type:
             if not isinstance(indexer, tuple):
                 indexer = self._tuplify(indexer)
@@ -192,14 +231,73 @@ def setter(item, v):
     def _align_series(self, indexer, ser):
         # indexer to assign Series can be tuple or scalar
         if isinstance(indexer, tuple):
+
+            aligners = [ not _is_null_slice(idx) for idx in indexer ]
+            single_aligner = sum(aligners) == 1
+            is_frame = self.obj.ndim == 2
+            is_panel = self.obj.ndim >= 3
+
+            # are we a single alignable value on a non-primary
+            # dim (e.g. panel: 1,2, or frame: 0) ?
+            # hence need to align to a single axis dimension
+            # rather that find all valid dims
+
+            # frame
+            if is_frame:
+                single_aligner = single_aligner and aligners[0]
+
+            # panel
+            elif is_panel:
+                single_aligner = single_aligner and (aligners[1] or aligners[2])
+
+            obj = self.obj
             for i, idx in enumerate(indexer):
-                ax = self.obj.axes[i]
+                ax = obj.axes[i]
+
+                # multiple aligners (or null slices)
                 if com._is_sequence(idx) or isinstance(idx, slice):
+                    if single_aligner and _is_null_slice(idx):
+                        continue
                     new_ix = ax[idx]
+                    if not is_list_like(new_ix):
+                        new_ix = Index([new_ix])
                     if ser.index.equals(new_ix):
                         return ser.values.copy()
                     return ser.reindex(new_ix).values
 
+                # 2 dims
+                elif single_aligner and is_frame:
+
+                    # reindex along index
+                    ax = self.obj.axes[1]
+                    if ser.index.equals(ax):
+                        return ser.values.copy()
+                    return ser.reindex(ax).values
+
+                # >2 dims
+                elif single_aligner:
+
+                    broadcast = []
+                    for n, labels in enumerate(self.obj._get_plane_axes(i)):
+
+                        # reindex along the matching dimensions
+                        if len(labels & ser.index):
+                            ser = ser.reindex(labels)
+                        else:
+                            broadcast.append((n,len(labels)))
+
+                    # broadcast along other dims
+                    ser = ser.values.copy()
+                    for (axis,l) in broadcast:
+                        shape = [ -1 ] * (len(broadcast)+1)
+                        shape[axis] = l
+                        ser = np.tile(ser,l).reshape(shape)
+
+                    if self.obj.ndim == 3:
+                        ser = ser.T
+
+                    return ser
+
         elif np.isscalar(indexer):
             ax = self.obj._get_axis(1)
 
@@ -521,7 +619,7 @@ def _reindex(keys, level=None):
 
                 return result
 
-    def _convert_to_indexer(self, obj, axis=0):
+    def _convert_to_indexer(self, obj, axis=0, is_setter=False):
         """
         Convert indexing key into something we can use to do actual fancy
         indexing on an ndarray
@@ -639,7 +737,14 @@ def _convert_to_indexer(self, obj, axis=0):
                 return indexer
 
         else:
-            return labels.get_loc(obj)
+            try:
+                return labels.get_loc(obj)
+            except (KeyError):
+
+                # allow a not found key only if we are a setter
+                if np.isscalar(obj) and is_setter:
+                    return { 'key' : obj }
+                raise
 
     def _tuplify(self, loc):
         tup = [slice(None, None) for _ in range(self.ndim)]
@@ -877,7 +982,7 @@ def _getitem_axis(self, key, axis=0):
 
             return self._get_loc(key,axis=axis)
 
-    def _convert_to_indexer(self, obj, axis=0):
+    def _convert_to_indexer(self, obj, axis=0, is_setter=False):
         """ much simpler as we only have to deal with our valid types """
         if self._has_valid_type(obj,axis):
             return obj
@@ -1028,6 +1133,12 @@ def _slice(self, indexer, axis=0):
         return self.obj._get_values(indexer)
 
     def _setitem_with_indexer(self, indexer, value):
+
+        # need to delegate to the super setter
+        if isinstance(indexer, dict):
+            return super(_SeriesIndexer, self)._setitem_with_indexer(indexer, value)
+
+        # fast access
         self.obj._set_values(indexer, value)
 
 def _check_bool_indexer(ax, key):
@@ -1053,6 +1164,21 @@ def _check_bool_indexer(ax, key):
     return result
 
 
+def _convert_missing_indexer(indexer):
+    """ reverse convert a missing indexer, which is a dict
+        return the scalar indexer and a boolean indicating if we converted """
+
+    if isinstance(indexer, dict):
+
+        # a missing key (but not a tuple indexer)
+        indexer = indexer['key']
+
+        if isinstance(indexer, bool):
+            raise KeyError("cannot use a single bool to index into setitem")
+        return indexer, True
+
+    return indexer, False
+
 def _maybe_convert_indices(indices, n):
     """ if we have negative indicies, translate to postive here
         if have indicies that are out-of-bounds, raise an IndexError """

pandas/core/internals.py

@@ -2647,7 +2647,7 @@ def reindex_axis(self, new_axis, indexer=None, method=None, axis=0, fill_value=N
             if method is not None or limit is not None:
                 return self.reindex_axis0_with_method(new_axis, indexer=indexer,
                                                       method=method, fill_value=fill_value, limit=limit, copy=copy)
-            return self.reindex_items(new_axis, copy=copy, fill_value=fill_value)
+            return self.reindex_items(new_axis, indexer=indexer, copy=copy, fill_value=fill_value)
 
         new_axis, indexer = cur_axis.reindex(
             new_axis, method, copy_if_needed=True)
@@ -2709,7 +2709,7 @@ def _reindex_indexer_items(self, new_items, indexer, fill_value):
 
         return self.__class__(new_blocks, new_axes)
 
-    def reindex_items(self, new_items, copy=True, fill_value=None):
+    def reindex_items(self, new_items, indexer=None, copy=True, fill_value=None):
         """
 
         """
@@ -2719,8 +2719,8 @@ def reindex_items(self, new_items, copy=True, fill_value=None):
             data = data.consolidate()
             return data.reindex_items(new_items, copy=copy, fill_value=fill_value)
 
-        # TODO: this part could be faster (!)
-        new_items, indexer = self.items.reindex(new_items, copy_if_needed=True)
+        if indexer is None:
+            new_items, indexer = self.items.reindex(new_items, copy_if_needed=True)
         new_axes = [new_items] + self.axes[1:]
 
         # could have so me pathological (MultiIndex) issues here

pandas/tests/test_frame.py

@@ -866,9 +866,11 @@ def test_fancy_index_int_labels_exceptions(self):
         self.assertRaises(KeyError,
                           self.frame.ix.__setitem__,
                           (slice(None, None), ['E']), 1)
-        self.assertRaises(KeyError,
-                          self.frame.ix.__setitem__,
-                          (slice(None, None), 'E'), 1)
+
+        # partial setting now allows this GH2578
+        #self.assertRaises(KeyError,
+        #                  self.frame.ix.__setitem__,
+        #                  (slice(None, None), 'E'), 1)
 
     def test_setitem_fancy_mixed_2d(self):
         self.mixed_frame.ix[:5, ['C', 'B', 'A']] = 5