Merge remote-tracking branch 'upstream/master' into bisect

Author: simonjayhawkins

ci/deps/azure-38.yaml

@@ -17,4 +17,4 @@ dependencies:
   - python-dateutil
   - nomkl
   - pytz
-  - tabulate==0.8.3
+  - tabulate==0.8.7

doc/source/getting_started/install.rst

@@ -283,7 +283,7 @@ pyreadstat                                   SPSS files (.sav) reading
 pyxlsb                    1.0.6              Reading for xlsb files
 qtpy                                         Clipboard I/O
 s3fs                      0.4.0              Amazon S3 access
-tabulate                  0.8.3              Printing in Markdown-friendly format (see `tabulate`_)
+tabulate                  0.8.7              Printing in Markdown-friendly format (see `tabulate`_)
 xarray                    0.12.3             pandas-like API for N-dimensional data
 xclip                                        Clipboard I/O on linux
 xlrd                      1.2.0              Excel reading

doc/source/reference/indexing.rst

@@ -290,6 +290,7 @@ MultiIndex properties
    MultiIndex.codes
    MultiIndex.nlevels
    MultiIndex.levshape
+   MultiIndex.dtypes
 
 MultiIndex components
 ~~~~~~~~~~~~~~~~~~~~~

doc/source/whatsnew/v1.2.0.rst

@@ -308,6 +308,7 @@ Other enhancements
 - Improved error reporting for subsetting columns of a :class:`.DataFrameGroupBy` with ``axis=1`` (:issue:`37725`)
 - Implement method ``cross`` for :meth:`DataFrame.merge` and :meth:`DataFrame.join` (:issue:`5401`)
 - When :func:`read_csv/sas/json` are called with ``chuncksize``/``iterator`` they can be used in a ``with`` statement as they return context-managers (:issue:`38225`)
+- Augmented the list of named colors available for styling Excel exports, enabling all of CSS4 colors (:issue:`38247`)
 
 .. ---------------------------------------------------------------------------
 
@@ -606,6 +607,7 @@ Datetimelike
 - Bug in :meth:`Series.isin` with ``datetime64[ns]`` dtype and :meth:`.DatetimeIndex.isin` failing to consider timezone-aware and timezone-naive datetimes as always different (:issue:`35728`)
 - Bug in :meth:`Series.isin` with ``PeriodDtype`` dtype and :meth:`PeriodIndex.isin` failing to consider arguments with different ``PeriodDtype`` as always different (:issue:`37528`)
 - Bug in :class:`Period` constructor now correctly handles nanoseconds in the ``value`` argument (:issue:`34621` and :issue:`17053`)
+- Bug in :meth:`DataFrame.first` and :meth:`Series.first` returning two months for offset one month when first day is last calendar day (:issue:`29623`)
 
 Timedelta
 ^^^^^^^^^

doc/source/whatsnew/v1.3.0.rst

@@ -19,7 +19,7 @@ Enhancements
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
 
--
+- Added :meth:`MultiIndex.dtypes` (:issue:`37062`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -37,7 +37,76 @@ These are bug fixes that might have notable behavior changes.
 
 Increased minimum versions for dependencies
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
+Some minimum supported versions of dependencies were updated.
+If installed, we now require:
+
++-----------------+-----------------+----------+---------+
+| Package         | Minimum Version | Required | Changed |
++=================+=================+==========+=========+
+| numpy           | 1.16.5          |    X     |         |
++-----------------+-----------------+----------+---------+
+| pytz            | 2017.3          |    X     |         |
++-----------------+-----------------+----------+---------+
+| python-dateutil | 2.7.3           |    X     |         |
++-----------------+-----------------+----------+---------+
+| bottleneck      | 1.2.1           |          |         |
++-----------------+-----------------+----------+---------+
+| numexpr         | 2.6.8           |          |         |
++-----------------+-----------------+----------+---------+
+| pytest (dev)    | 5.0.1           |          |         |
++-----------------+-----------------+----------+---------+
+| mypy (dev)      | 0.782           |          |         |
++-----------------+-----------------+----------+---------+
+
+For `optional libraries <https://dev.pandas.io/docs/install.html#dependencies>`_ the general recommendation is to use the latest version.
+The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
+Optional libraries below the lowest tested version may still work, but are not considered supported.
+
++-----------------+-----------------+---------+
+| Package         | Minimum Version | Changed |
++=================+=================+=========+
+| beautifulsoup4  | 4.6.0           |         |
++-----------------+-----------------+---------+
+| fastparquet     | 0.3.2           |         |
++-----------------+-----------------+---------+
+| fsspec          | 0.7.4           |         |
++-----------------+-----------------+---------+
+| gcsfs           | 0.6.0           |         |
++-----------------+-----------------+---------+
+| lxml            | 4.3.0           |         |
++-----------------+-----------------+---------+
+| matplotlib      | 2.2.3           |         |
++-----------------+-----------------+---------+
+| numba           | 0.46.0          |         |
++-----------------+-----------------+---------+
+| openpyxl        | 2.6.0           |         |
++-----------------+-----------------+---------+
+| pyarrow         | 0.15.0          |         |
++-----------------+-----------------+---------+
+| pymysql         | 0.7.11          |         |
++-----------------+-----------------+---------+
+| pytables        | 3.5.1           |         |
++-----------------+-----------------+---------+
+| s3fs            | 0.4.0           |         |
++-----------------+-----------------+---------+
+| scipy           | 1.2.0           |         |
++-----------------+-----------------+---------+
+| sqlalchemy      | 1.2.8           |         |
++-----------------+-----------------+---------+
+| tabulate        | 0.8.7           |    X    |
++-----------------+-----------------+---------+
+| xarray          | 0.12.0          |         |
++-----------------+-----------------+---------+
+| xlrd            | 1.2.0           |         |
++-----------------+-----------------+---------+
+| xlsxwriter      | 1.0.2           |         |
++-----------------+-----------------+---------+
+| xlwt            | 1.3.0           |         |
++-----------------+-----------------+---------+
+| pandas-gbq      | 0.12.0          |         |
++-----------------+-----------------+---------+
+
+See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more.
 
 .. _whatsnew_130.api.other:
 
@@ -86,7 +155,7 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
-
+- Bug in :class:`DataFrame` and :class:`Series` constructors sometimes dropping nanoseconds from :class:`Timestamp` (resp. :class:`Timedelta`) ``data``, with ``dtype=datetime64[ns]`` (resp. ``timedelta64[ns]``) (:issue:`38032`)
 -
 -
 
@@ -127,7 +196,9 @@ Interval
 
 Indexing
 ^^^^^^^^
-
+- Bug in :meth:`CategoricalIndex.get_indexer` failing to raise ``InvalidIndexError`` when non-unique (:issue:`38372`)
+- Bug in inserting many new columns into a :class:`DataFrame` causing incorrect subsequent indexing behavior (:issue:`38380`)
+- Bug in :meth:`DataFrame.iloc.__setitem__` and :meth:`DataFrame.loc.__setitem__` with mixed dtypes when setting with a dictionary value (:issue:`38335`)
 -
 -
 
@@ -146,7 +217,7 @@ MultiIndex
 I/O
 ^^^
 
--
+- Bug in :func:`read_csv` raising ``TypeError`` when ``names`` and ``parse_dates`` is specified for ``engine="c"`` (:issue:`33699`)
 -
 
 Period

pandas/_libs/algos.pyx

@@ -824,7 +824,7 @@ def rank_1d(
         if values.dtype != np.object_:
             values = values.astype('O')
     else:
-        values = np.asarray(in_arr)
+        values = np.asarray(in_arr).copy()
 
     keep_na = na_option == 'keep'
 
@@ -835,11 +835,6 @@ def rank_1d(
     elif rank_t is int64_t:
         mask = values == NPY_NAT
 
-        # create copy in case of NPY_NAT
-        # values are mutated inplace
-        if mask.any():
-            values = values.copy()
-
     # double sort first by mask and then by values to ensure nan values are
     # either at the beginning or the end. mask/(~mask) controls padding at
     # tail or the head

pandas/compat/_optional.py

@@ -24,7 +24,7 @@
     "scipy": "1.2.0",
     "sqlalchemy": "1.2.8",
     "tables": "3.5.1",
-    "tabulate": "0.8.3",
+    "tabulate": "0.8.7",
     "xarray": "0.12.3",
     "xlrd": "1.2.0",
     "xlwt": "1.3.0",

pandas/core/algorithms.py

@@ -50,7 +50,7 @@
 from pandas.core.dtypes.generic import (
     ABCDatetimeArray,
     ABCExtensionArray,
-    ABCIndexClass,
+    ABCIndex,
     ABCMultiIndex,
     ABCRangeIndex,
     ABCSeries,
@@ -63,6 +63,7 @@
 
 if TYPE_CHECKING:
     from pandas import Categorical, DataFrame, Index, Series
+    from pandas.core.arrays import DatetimeArray, TimedeltaArray
 
 _shared_docs: Dict[str, str] = {}
 
@@ -215,7 +216,7 @@ def _reconstruct_data(
         values = values.astype(dtype, copy=False)
 
         # we only support object dtypes bool Index
-        if isinstance(original, ABCIndexClass):
+        if isinstance(original, ABCIndex):
             values = values.astype(object, copy=False)
     elif dtype is not None:
         if is_datetime64_dtype(dtype):
@@ -437,9 +438,7 @@ def isin(comps: AnyArrayLike, values: AnyArrayLike) -> np.ndarray:
             f"to isin(), you passed a [{type(values).__name__}]"
         )
 
-    if not isinstance(
-        values, (ABCIndexClass, ABCSeries, ABCExtensionArray, np.ndarray)
-    ):
+    if not isinstance(values, (ABCIndex, ABCSeries, ABCExtensionArray, np.ndarray)):
         values = _ensure_arraylike(list(values))
     elif isinstance(values, ABCMultiIndex):
         # Avoid raising in extract_array
@@ -700,7 +699,7 @@ def factorize(
         and values.freq is not None
     ):
         codes, uniques = values.factorize(sort=sort)
-        if isinstance(original, ABCIndexClass):
+        if isinstance(original, ABCIndex):
             uniques = original._shallow_copy(uniques, name=None)
         elif isinstance(original, ABCSeries):
             from pandas import Index
@@ -739,8 +738,11 @@ def factorize(
     uniques = _reconstruct_data(uniques, dtype, original)
 
     # return original tenor
-    if isinstance(original, ABCIndexClass):
+    if isinstance(original, ABCIndex):
         if original.dtype.kind in ["m", "M"] and isinstance(uniques, np.ndarray):
+            original._data = cast(
+                "Union[DatetimeArray, TimedeltaArray]", original._data
+            )
             uniques = type(original._data)._simple_new(uniques, dtype=original.dtype)
         uniques = original._shallow_copy(uniques, name=None)
     elif isinstance(original, ABCSeries):

pandas/core/arrays/__init__.py

@@ -7,7 +7,7 @@
 from pandas.core.arrays.categorical import Categorical
 from pandas.core.arrays.datetimes import DatetimeArray
 from pandas.core.arrays.floating import FloatingArray
-from pandas.core.arrays.integer import IntegerArray, integer_array
+from pandas.core.arrays.integer import IntegerArray
 from pandas.core.arrays.interval import IntervalArray
 from pandas.core.arrays.masked import BaseMaskedArray
 from pandas.core.arrays.numpy_ import PandasArray, PandasDtype
@@ -26,7 +26,6 @@
     "DatetimeArray",
     "FloatingArray",
     "IntegerArray",
-    "integer_array",
     "IntervalArray",
     "PandasArray",
     "PandasDtype",

pandas/core/arrays/base.py

@@ -41,7 +41,7 @@
     pandas_dtype,
 )
 from pandas.core.dtypes.dtypes import ExtensionDtype
-from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries
+from pandas.core.dtypes.generic import ABCDataFrame, ABCIndex, ABCSeries
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import ops
@@ -1361,7 +1361,7 @@ def convert_values(param):
                     ovalues = [param] * len(self)
                 return ovalues
 
-            if isinstance(other, (ABCSeries, ABCIndexClass, ABCDataFrame)):
+            if isinstance(other, (ABCSeries, ABCIndex, ABCDataFrame)):
                 # rely on pandas to unbox and dispatch to us
                 return NotImplemented
 

pandas/core/arrays/categorical.py

@@ -38,7 +38,7 @@
     needs_i8_conversion,
 )
 from pandas.core.dtypes.dtypes import CategoricalDtype
-from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries
+from pandas.core.dtypes.generic import ABCIndex, ABCSeries
 from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna, notna
 
 from pandas.core import ops
@@ -321,7 +321,7 @@ def __init__(
         if is_categorical_dtype(values):
             if dtype.categories is None:
                 dtype = CategoricalDtype(values.categories, dtype.ordered)
-        elif not isinstance(values, (ABCIndexClass, ABCSeries)):
+        elif not isinstance(values, (ABCIndex, ABCSeries)):
             # sanitize_array coerces np.nan to a string under certain versions
             # of numpy
             values = maybe_infer_to_datetimelike(values, convert_dates=True)
@@ -2514,7 +2514,7 @@ def _get_codes_for_values(values, categories) -> np.ndarray:
         values = ensure_object(values)
         categories = ensure_object(categories)
 
-    if isinstance(categories, ABCIndexClass):
+    if isinstance(categories, ABCIndex):
         return coerce_indexer_dtype(categories.get_indexer_for(values), categories)
 
     # Only hit here when we've already coerced to object dtypee.

pandas/core/arrays/datetimes.py

@@ -44,7 +44,7 @@
     pandas_dtype,
 )
 from pandas.core.dtypes.dtypes import DatetimeTZDtype
-from pandas.core.dtypes.generic import ABCIndexClass, ABCPandasArray, ABCSeries
+from pandas.core.dtypes.generic import ABCIndex, ABCPandasArray, ABCSeries
 from pandas.core.dtypes.missing import isna
 
 from pandas.core.algorithms import checked_add_with_arr
@@ -216,7 +216,7 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps):
     _freq = None
 
     def __init__(self, values, dtype=DT64NS_DTYPE, freq=None, copy=False):
-        if isinstance(values, (ABCSeries, ABCIndexClass)):
+        if isinstance(values, (ABCSeries, ABCIndex)):
             values = values._values
 
         inferred_freq = getattr(values, "_freq", None)
@@ -1947,7 +1947,7 @@ def sequence_to_dt64ns(
     # if dtype has an embedded tz, capture it
     tz = validate_tz_from_dtype(dtype, tz)
 
-    if isinstance(data, ABCIndexClass):
+    if isinstance(data, ABCIndex):
         if data.nlevels > 1:
             # Without this check, data._data below is None
             raise TypeError("Cannot create a DatetimeArray from a MultiIndex.")

pandas/core/arrays/floating.py

@@ -1,5 +1,5 @@
 import numbers
-from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union
+from typing import List, Optional, Tuple, Type
 import warnings
 
 import numpy as np
@@ -27,13 +27,10 @@
 from pandas.core.tools.numeric import to_numeric
 
 from .masked import BaseMaskedDtype
-from .numeric import NumericArray
+from .numeric import NumericArray, NumericDtype
 
-if TYPE_CHECKING:
-    import pyarrow
 
-
-class FloatingDtype(BaseMaskedDtype):
+class FloatingDtype(NumericDtype):
     """
     An ExtensionDtype to hold a single size of floating dtype.
 
@@ -72,34 +69,6 @@ def _get_common_dtype(self, dtypes: List[DtypeObj]) -> Optional[DtypeObj]:
             return FLOAT_STR_TO_DTYPE[str(np_dtype)]
         return None
 
-    def __from_arrow__(
-        self, array: Union["pyarrow.Array", "pyarrow.ChunkedArray"]
-    ) -> "FloatingArray":
-        """
-        Construct FloatingArray from pyarrow Array/ChunkedArray.
-        """
-        import pyarrow
-
-        from pandas.core.arrays._arrow_utils import pyarrow_array_to_numpy_and_mask
-
-        pyarrow_type = pyarrow.from_numpy_dtype(self.type)
-        if not array.type.equals(pyarrow_type):
-            array = array.cast(pyarrow_type)
-
-        if isinstance(array, pyarrow.Array):
-            chunks = [array]
-        else:
-            # pyarrow.ChunkedArray
-            chunks = array.chunks
-
-        results = []
-        for arr in chunks:
-            data, mask = pyarrow_array_to_numpy_and_mask(arr, dtype=self.type)
-            float_arr = FloatingArray(data.copy(), ~mask, copy=False)
-            results.append(float_arr)
-
-        return FloatingArray._concat_same_type(results)
-
 
 def coerce_to_array(
     values, dtype=None, mask=None, copy: bool = False