From 9ae69e38745762113a2851ff2f8315e92f1d8fdb Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 5 Jun 2019 20:38:39 +0100 Subject: [PATCH 1/7] BUG: .isin on datetimelike indexes do validate input of level parameter --- doc/source/whatsnew/v0.25.0.rst | 1 + pandas/core/indexes/datetimelike.py | 3 +++ pandas/tests/indexes/conftest.py | 39 ++++++++++++++++++++++++++++- pandas/tests/indexes/test_base.py | 26 +++++++++---------- 4 files changed, 55 insertions(+), 14 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index 267e34efc946f..67bb8f4ab48bd 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -533,6 +533,7 @@ Datetimelike - Bug in :func:`to_datetime` which does not replace the invalid argument with ``NaT`` when error is set to coerce (:issue:`26122`) - Bug in adding :class:`DateOffset` with nonzero month to :class:`DatetimeIndex` would raise ``ValueError`` (:issue:`26258`) - Bug in :func:`to_datetime` which raises unhandled ``OverflowError`` when called with mix of invalid dates and ``NaN`` values with ``format='%Y%m%d'`` and ``error='coerce'`` (:issue:`25512`) +- Bug in :meth:`isin` for datetimelike indexes; :class:`DatetimeIndex`, :class:`TimedeltaIndex` and :class:`PeriodIndex` where the ``levels`` parmeter was ignored. (:issue:`26675`) Timedelta ^^^^^^^^^ diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 092cec00228cd..fb92f918f9203 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -522,6 +522,9 @@ def isin(self, values, level=None): ------- is_contained : ndarray (boolean dtype) """ + if level is not None: + self._validate_index_level(level) + if not isinstance(values, type(self)): try: values = type(self)(values) diff --git a/pandas/tests/indexes/conftest.py b/pandas/tests/indexes/conftest.py index 83f1f22b158b1..18aad67fbbceb 100644 --- a/pandas/tests/indexes/conftest.py +++ b/pandas/tests/indexes/conftest.py @@ -2,7 +2,7 @@ import pytest import pandas as pd -from pandas.core.indexes.api import Index, MultiIndex +from pandas.core.indexes.api import Index, MultiIndex, RangeIndex, PeriodIndex import pandas.util.testing as tm indices_list = [tm.makeUnicodeIndex(100), @@ -47,3 +47,40 @@ def zero(request): # For testing division by (or of) zero for Index with length 5, this # gives several scalar-zeros and length-5 vector-zeros return request.param + + +def _get_subclasses(cls): + for subclass in cls.__subclasses__(): + yield from _get_subclasses(subclass) + yield subclass + + +all_indexes = [index for index in ([Index] + list(set(_get_subclasses(Index)))) + if getattr(pd, index.__name__, None) is not None] + + +@pytest.fixture(params=all_indexes) +def all_index_types(request): + """ + A Fixture for all indexes types. Index and subclasses (includes ABCs). + """ + return request.param + + +@pytest.fixture +def all_index_empty(all_index_types): + """ + A Fixture for empty instances of all indexes types (except ABCs). + """ + cls = all_index_types + if issubclass(cls, RangeIndex): + return cls(0, name='foo') + elif issubclass(cls, MultiIndex): + return cls.from_arrays([[], []], names=['foo', 'bar']) + elif issubclass(cls, PeriodIndex): + return cls([], freq='M', name='foo') + else: + try: + return cls([], name='foo') + except AttributeError: + pytest.skip("{} is an ABC.".format(cls.__name__)) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index 7e70d77ea70fc..c57a9f783e42e 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -1791,22 +1791,22 @@ def test_isin_level_kwarg(self, level, index): tm.assert_numpy_array_equal(expected, index.isin(values, level='foobar')) - @pytest.mark.parametrize("level", [1, 10, -2]) - @pytest.mark.parametrize("index", [ - Index(['qux', 'baz', 'foo', 'bar']), - # Float64Index overrides isin, so must be checked separately - Float64Index([1.0, 2.0, 3.0, 4.0])]) - def test_isin_level_kwarg_raises_bad_index(self, level, index): + @pytest.mark.parametrize("level", [2, 10, -3]) + def test_isin_level_kwarg_bad_level_raises(self, level, all_index_empty): + index = all_index_empty with pytest.raises(IndexError, match='Too many levels'): index.isin([], level=level) - @pytest.mark.parametrize("level", [1.0, 'foobar', 'xyzzy', np.nan]) - @pytest.mark.parametrize("index", [ - Index(['qux', 'baz', 'foo', 'bar']), - Float64Index([1.0, 2.0, 3.0, 4.0])]) - def test_isin_level_kwarg_raises_key(self, level, index): - with pytest.raises(KeyError, match='must be same as name'): - index.isin([], level=level) + @pytest.mark.parametrize("label", [1.0, 'foobar', 'xyzzy', np.nan]) + def test_isin_level_kwarg_bad_label_raises( + self, label, all_index_empty): + index = all_index_empty + if isinstance(index, pd.MultiIndex): + msg = "'Level {} not found'" + else: + msg = r"'Level {} must be same as name \(foo\)'" + with pytest.raises(KeyError, match=msg.format(label)): + index.isin([], level=label) @pytest.mark.parametrize("empty", [[], Series(), np.array([])]) def test_isin_empty(self, empty): From 24bbb24ba38e67a593195d6385163416141e60f7 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 5 Jun 2019 20:57:35 +0100 Subject: [PATCH 2/7] remove unused code from conftest.py --- pandas/tests/indexes/conftest.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pandas/tests/indexes/conftest.py b/pandas/tests/indexes/conftest.py index 18aad67fbbceb..dbefdf4e16df3 100644 --- a/pandas/tests/indexes/conftest.py +++ b/pandas/tests/indexes/conftest.py @@ -62,7 +62,7 @@ def _get_subclasses(cls): @pytest.fixture(params=all_indexes) def all_index_types(request): """ - A Fixture for all indexes types. Index and subclasses (includes ABCs). + A Fixture for all indexes types. Index and subclasses in pandas namespace. """ return request.param @@ -70,7 +70,7 @@ def all_index_types(request): @pytest.fixture def all_index_empty(all_index_types): """ - A Fixture for empty instances of all indexes types (except ABCs). + A Fixture for empty instances of all indexes types in pandas namespace. """ cls = all_index_types if issubclass(cls, RangeIndex): @@ -80,7 +80,4 @@ def all_index_empty(all_index_types): elif issubclass(cls, PeriodIndex): return cls([], freq='M', name='foo') else: - try: - return cls([], name='foo') - except AttributeError: - pytest.skip("{} is an ABC.".format(cls.__name__)) + return cls([], name='foo') From 1c8883921cac00462bc001cd92aa7ba5f66f9267 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Thu, 6 Jun 2019 17:59:02 +0100 Subject: [PATCH 3/7] sort indexes to fix mutlicore pytest failures (and isort) --- pandas/tests/indexes/conftest.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexes/conftest.py b/pandas/tests/indexes/conftest.py index dbefdf4e16df3..571e32a5900b7 100644 --- a/pandas/tests/indexes/conftest.py +++ b/pandas/tests/indexes/conftest.py @@ -1,8 +1,10 @@ +from operator import attrgetter + import numpy as np import pytest import pandas as pd -from pandas.core.indexes.api import Index, MultiIndex, RangeIndex, PeriodIndex +from pandas.core.indexes.api import Index, MultiIndex, PeriodIndex, RangeIndex import pandas.util.testing as tm indices_list = [tm.makeUnicodeIndex(100), @@ -55,7 +57,10 @@ def _get_subclasses(cls): yield subclass -all_indexes = [index for index in ([Index] + list(set(_get_subclasses(Index)))) +all_indexes_inc_abc = [Index] + list(set(_get_subclasses(Index))) +all_indexes_inc_abc_sorted = sorted(all_indexes_inc_abc, + key=attrgetter('__name__')) +all_indexes = [index for index in all_indexes_inc_abc_sorted if getattr(pd, index.__name__, None) is not None] From 2d201b0352d25ebb8a7feac7cc58e09aa92800d9 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Thu, 6 Jun 2019 19:19:14 +0100 Subject: [PATCH 4/7] fix trailing whitespace in rst file --- doc/source/whatsnew/v0.25.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index 04f4ed5a58833..802fc9a6e511e 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -538,7 +538,7 @@ Datetimelike - Bug in :func:`to_datetime` which does not replace the invalid argument with ``NaT`` when error is set to coerce (:issue:`26122`) - Bug in adding :class:`DateOffset` with nonzero month to :class:`DatetimeIndex` would raise ``ValueError`` (:issue:`26258`) - Bug in :func:`to_datetime` which raises unhandled ``OverflowError`` when called with mix of invalid dates and ``NaN`` values with ``format='%Y%m%d'`` and ``error='coerce'`` (:issue:`25512`) -- Bug in :meth:`isin` for datetimelike indexes; :class:`DatetimeIndex`, :class:`TimedeltaIndex` and :class:`PeriodIndex` where the ``levels`` parmeter was ignored. (:issue:`26675`) +- Bug in :meth:`isin` for datetimelike indexes; :class:`DatetimeIndex`, :class:`TimedeltaIndex` and :class:`PeriodIndex` where the ``levels`` parmeter was ignored. (:issue:`26675`) - Bug in :func:`to_datetime` which raises ``TypeError`` for ``format='%Y%m%d'`` when called for invalid integer dates with length >= 6 digits with ``errors='ignore'`` Timedelta From 4bcdf6367f4fe57e05f34fa9472c34ab66ce2f65 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Thu, 6 Jun 2019 21:50:49 +0100 Subject: [PATCH 5/7] simplify expression --- pandas/tests/indexes/conftest.py | 2 +- pandas/tests/indexes/test_base.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pandas/tests/indexes/conftest.py b/pandas/tests/indexes/conftest.py index 571e32a5900b7..531740a227609 100644 --- a/pandas/tests/indexes/conftest.py +++ b/pandas/tests/indexes/conftest.py @@ -61,7 +61,7 @@ def _get_subclasses(cls): all_indexes_inc_abc_sorted = sorted(all_indexes_inc_abc, key=attrgetter('__name__')) all_indexes = [index for index in all_indexes_inc_abc_sorted - if getattr(pd, index.__name__, None) is not None] + if getattr(pd, index.__name__, False)] @pytest.fixture(params=all_indexes) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index c57a9f783e42e..0f14341b91402 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -1798,8 +1798,7 @@ def test_isin_level_kwarg_bad_level_raises(self, level, all_index_empty): index.isin([], level=level) @pytest.mark.parametrize("label", [1.0, 'foobar', 'xyzzy', np.nan]) - def test_isin_level_kwarg_bad_label_raises( - self, label, all_index_empty): + def test_isin_level_kwarg_bad_label_raises(self, label, all_index_empty): index = all_index_empty if isinstance(index, pd.MultiIndex): msg = "'Level {} not found'" From d8b2c377cd70b93fa98a2cf68f267c19cdd1216e Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 9 Jun 2019 21:19:50 +0100 Subject: [PATCH 6/7] remove unnecessary complexity --- pandas/tests/indexes/conftest.py | 41 +------------------------------ pandas/tests/indexes/test_base.py | 12 +++++---- 2 files changed, 8 insertions(+), 45 deletions(-) diff --git a/pandas/tests/indexes/conftest.py b/pandas/tests/indexes/conftest.py index 531740a227609..83f1f22b158b1 100644 --- a/pandas/tests/indexes/conftest.py +++ b/pandas/tests/indexes/conftest.py @@ -1,10 +1,8 @@ -from operator import attrgetter - import numpy as np import pytest import pandas as pd -from pandas.core.indexes.api import Index, MultiIndex, PeriodIndex, RangeIndex +from pandas.core.indexes.api import Index, MultiIndex import pandas.util.testing as tm indices_list = [tm.makeUnicodeIndex(100), @@ -49,40 +47,3 @@ def zero(request): # For testing division by (or of) zero for Index with length 5, this # gives several scalar-zeros and length-5 vector-zeros return request.param - - -def _get_subclasses(cls): - for subclass in cls.__subclasses__(): - yield from _get_subclasses(subclass) - yield subclass - - -all_indexes_inc_abc = [Index] + list(set(_get_subclasses(Index))) -all_indexes_inc_abc_sorted = sorted(all_indexes_inc_abc, - key=attrgetter('__name__')) -all_indexes = [index for index in all_indexes_inc_abc_sorted - if getattr(pd, index.__name__, False)] - - -@pytest.fixture(params=all_indexes) -def all_index_types(request): - """ - A Fixture for all indexes types. Index and subclasses in pandas namespace. - """ - return request.param - - -@pytest.fixture -def all_index_empty(all_index_types): - """ - A Fixture for empty instances of all indexes types in pandas namespace. - """ - cls = all_index_types - if issubclass(cls, RangeIndex): - return cls(0, name='foo') - elif issubclass(cls, MultiIndex): - return cls.from_arrays([[], []], names=['foo', 'bar']) - elif issubclass(cls, PeriodIndex): - return cls([], freq='M', name='foo') - else: - return cls([], name='foo') diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index 0f14341b91402..1de20dc765655 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -1792,17 +1792,19 @@ def test_isin_level_kwarg(self, level, index): index.isin(values, level='foobar')) @pytest.mark.parametrize("level", [2, 10, -3]) - def test_isin_level_kwarg_bad_level_raises(self, level, all_index_empty): - index = all_index_empty + def test_isin_level_kwarg_bad_level_raises(self, level, indices): + index = indices with pytest.raises(IndexError, match='Too many levels'): index.isin([], level=level) @pytest.mark.parametrize("label", [1.0, 'foobar', 'xyzzy', np.nan]) - def test_isin_level_kwarg_bad_label_raises(self, label, all_index_empty): - index = all_index_empty - if isinstance(index, pd.MultiIndex): + def test_isin_level_kwarg_bad_label_raises(self, label, indices): + index = indices + if isinstance(index, MultiIndex): + index = index.rename(['foo', 'bar']) msg = "'Level {} not found'" else: + index = index.rename('foo') msg = r"'Level {} must be same as name \(foo\)'" with pytest.raises(KeyError, match=msg.format(label)): index.isin([], level=label) From 84e8cc49a9809a609010767f2cd0b9176806d546 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 9 Jun 2019 21:23:11 +0100 Subject: [PATCH 7/7] fix typo --- doc/source/whatsnew/v0.25.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index eda41de3eef02..df22a21196dab 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -540,7 +540,7 @@ Datetimelike - Bug in :func:`to_datetime` which does not replace the invalid argument with ``NaT`` when error is set to coerce (:issue:`26122`) - Bug in adding :class:`DateOffset` with nonzero month to :class:`DatetimeIndex` would raise ``ValueError`` (:issue:`26258`) - Bug in :func:`to_datetime` which raises unhandled ``OverflowError`` when called with mix of invalid dates and ``NaN`` values with ``format='%Y%m%d'`` and ``error='coerce'`` (:issue:`25512`) -- Bug in :meth:`isin` for datetimelike indexes; :class:`DatetimeIndex`, :class:`TimedeltaIndex` and :class:`PeriodIndex` where the ``levels`` parmeter was ignored. (:issue:`26675`) +- Bug in :meth:`isin` for datetimelike indexes; :class:`DatetimeIndex`, :class:`TimedeltaIndex` and :class:`PeriodIndex` where the ``levels`` parameter was ignored. (:issue:`26675`) - Bug in :func:`to_datetime` which raises ``TypeError`` for ``format='%Y%m%d'`` when called for invalid integer dates with length >= 6 digits with ``errors='ignore'`` - Bug when comparing a :class:`PeriodIndex` against a zero-dimensional numpy array (:issue:`26689`)