-
-
Notifications
You must be signed in to change notification settings - Fork 18.5k
BUG: DataFrame.first_valid_index() fails if there is no valid entry. #17488
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4069,23 +4069,97 @@ def update(self, other, join='left', overwrite=True, filter_func=None, | |
# ---------------------------------------------------------------------- | ||
# Misc methods | ||
|
||
def _get_valid_indices(self): | ||
is_valid = self.count(1) > 0 | ||
return self.index[is_valid] | ||
|
||
def first_valid_index(self): | ||
""" | ||
Return label for first non-NA/null value | ||
Return index for first non-NA/null value. | ||
If all elements are non-NA/null, returns None. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Add these 2 last sentences to a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. add a |
||
Also returns None for empty DataFrame. | ||
|
||
Examples | ||
-------- | ||
|
||
When no null value in a DataFrame, returns first index. | ||
|
||
>>> df = DataFrame({'A': [1, 2, 3], 'B': ['a', 'b', 'f']}) | ||
>>> df.first_valid_index() | ||
0 | ||
|
||
When all elements in first row are null, returns second index. | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ideally would like to move these doc-strings to There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @jreback Note: I have no idea to share the doc-strings between |
||
>>> df = DataFrame({'A': [None, 2, 3], 'B': [None, 'b', 'f']}) | ||
>>> df.first_valid_index() | ||
1 | ||
|
||
When only part of elements in first row are null, recognized as valid. | ||
|
||
>>> df = DataFrame({'A': [1, 2, 3], 'B': [None, 'b', 'f']}) | ||
>>> df.first_valid_index() | ||
0 | ||
|
||
When all elements in a dataframe are null, returns None | ||
|
||
>>> df = DataFrame({'A': [None, None, None], 'B': [None, None, None]}) | ||
>>> df.first_valid_index() | ||
|
||
Returns None for empty DataFrame | ||
|
||
>>> df = DataFrame() | ||
>>> df.first_valid_index() | ||
|
||
""" | ||
if len(self) == 0: | ||
return None | ||
|
||
return self.index[self.count(1) > 0][0] | ||
valid_indices = self._get_valid_indices() | ||
return valid_indices[0] if len(valid_indices) else None | ||
|
||
def last_valid_index(self): | ||
""" | ||
Return label for last non-NA/null value | ||
Return index for last non-NA/null value. | ||
If all elements are non-NA/null, returns None. | ||
Also returns None for empty DataFrame. | ||
|
||
Examples | ||
-------- | ||
|
||
When no null value in a DataFrame, returns last index. | ||
|
||
>>> df = DataFrame({'A': [1, 2, 3], 'B': ['a', 'b', 'f']}) | ||
>>> df.last_valid_index() | ||
2 | ||
|
||
When all elements in last row are null, returns second from last index. | ||
|
||
>>> df = DataFrame({'A': [1, 2, None], 'B': ['a', 'b', None]}) | ||
>>> df.last_valid_index() | ||
1 | ||
|
||
When only part of elements in last row are null, recognized as valid. | ||
|
||
>>> df = DataFrame({'A': [1, 2, 3], 'B': ['a', 'b', None]}) | ||
>>> df.last_valid_index() | ||
2 | ||
|
||
When all elements in a dataframe are null, returns None | ||
|
||
>>> df = DataFrame({'A': [None, None, None], 'B': [None, None, None]}) | ||
>>> df.last_valid_index() | ||
|
||
Returns None for empty DataFrame | ||
|
||
>>> df = DataFrame() | ||
>>> df.last_valid_index() | ||
|
||
""" | ||
if len(self) == 0: | ||
return None | ||
|
||
return self.index[self.count(1) > 0][-1] | ||
valid_indices = self._get_valid_indices() | ||
return valid_indices[-1] if len(valid_indices) else None | ||
|
||
# ---------------------------------------------------------------------- | ||
# Data reshaping | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -440,6 +440,11 @@ def test_first_last_valid(self): | |
assert empty.last_valid_index() is None | ||
assert empty.first_valid_index() is None | ||
|
||
# GH17400 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. add a comment (no valid entries) |
||
frame[:] = nan | ||
assert frame.last_valid_index() is None | ||
assert frame.first_valid_index() is None | ||
|
||
def test_at_time_frame(self): | ||
rng = date_range('1/1/2000', '1/5/2000', freq='5min') | ||
ts = DataFrame(np.random.randn(len(rng), 2), index=rng) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
you can do
:meth:`DataFrame.first_valid_index`
and so on