Merge remote-tracking branch 'upstream/main' into dateoffset-bug

Author: rsm-23

.github/workflows/unit-tests.yml

@@ -230,7 +230,7 @@ jobs:
           /opt/python/cp39-cp39/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install -U pip wheel setuptools meson[ninja]==1.0.1 meson-python==0.13.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
@@ -268,7 +268,7 @@ jobs:
           /opt/python/cp39-cp39/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.0.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
 
@@ -340,7 +340,7 @@ jobs:
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install git+https://github.com/nedbat/coveragepy.git
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz cython hypothesis>=6.46.1 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
+          python -m pip install python-dateutil pytz cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
           python -m pip list
 
       - name: Build Pandas

.github/workflows/wheels.yml

@@ -140,7 +140,7 @@ jobs:
         shell: pwsh
         run: |
           $TST_CMD = @"
-          python -m pip install pytz six numpy python-dateutil tzdata>=2022.1 hypothesis>=6.46.1 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17;
+          python -m pip install pytz six numpy python-dateutil tzdata>=2022.1 hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17;
           python -m pip install --find-links=pandas\wheelhouse --no-index pandas;
           python -c `'import pandas as pd; pd.test()`';
           "@

asv_bench/benchmarks/multiindex_object.py

@@ -396,4 +396,30 @@ def time_putmask_all_different(self):
         self.midx.putmask(self.mask, self.midx_values_different)
 
 
+class Append:
+    params = ["datetime64[ns]", "int64", "string"]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        N1 = 1000
+        N2 = 500
+        left_level1 = range(N1)
+        right_level1 = range(N1, N1 + N1)
+
+        if dtype == "datetime64[ns]":
+            level2 = date_range(start="2000-01-01", periods=N2)
+        elif dtype == "int64":
+            level2 = range(N2)
+        elif dtype == "string":
+            level2 = tm.makeStringIndex(N2)
+        else:
+            raise NotImplementedError
+
+        self.left = MultiIndex.from_product([left_level1, level2])
+        self.right = MultiIndex.from_product([right_level1, level2])
+
+    def time_append(self, dtype):
+        self.left.append(self.right)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

ci/code_checks.sh

@@ -105,20 +105,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.errors.UnsupportedFunctionCall \
         pandas.test \
         pandas.NaT \
-        pandas.Timestamp.date \
-        pandas.Timestamp.dst \
-        pandas.Timestamp.isocalendar \
-        pandas.Timestamp.isoweekday \
-        pandas.Timestamp.strptime \
-        pandas.Timestamp.time \
-        pandas.Timestamp.timetuple \
-        pandas.Timestamp.timetz \
-        pandas.Timestamp.to_datetime64 \
-        pandas.Timestamp.toordinal \
-        pandas.Timestamp.tzname \
-        pandas.Timestamp.utcoffset \
-        pandas.Timestamp.utctimetuple \
-        pandas.Timestamp.weekday \
         pandas.arrays.TimedeltaArray \
         pandas.Period.asfreq \
         pandas.Period.now \

ci/deps/actions-310.yaml

@@ -11,7 +11,7 @@ dependencies:
   - meson-python=0.13.1
 
   # test dependencies
-  - pytest>=7.0.0
+  - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
   - pytest-asyncio>=0.17.0

ci/deps/actions-311-downstream_compat.yaml

@@ -12,7 +12,7 @@ dependencies:
   - meson-python=0.13.1
 
   # test dependencies
-  - pytest>=7.0.0
+  - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
   - pytest-asyncio>=0.17.0

ci/deps/actions-311-numpydev.yaml

@@ -10,7 +10,7 @@ dependencies:
   - meson-python=0.13.1
 
   # test dependencies
-  - pytest>=7.0.0
+  - pytest>=7.3.2
   - pytest-cov
   # Once pytest-cov > 4 comes out, unpin this
   # Right now, a DeprecationWarning related to rsyncdir

ci/deps/actions-311-pyarrownightly.yaml

@@ -11,7 +11,7 @@ dependencies:
   - meson-python=0.13.1
 
   # test dependencies
-  - pytest>=7.0.0
+  - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
   - hypothesis>=6.46.1

ci/deps/actions-311.yaml

@@ -11,7 +11,7 @@ dependencies:
   - meson-python=0.13.1
 
   # test dependencies
-  - pytest>=7.0.0
+  - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
   - pytest-asyncio>=0.17.0

ci/deps/actions-39-minimum_versions.yaml

@@ -13,7 +13,7 @@ dependencies:
   - meson-python=0.13.1
 
   # test dependencies
-  - pytest>=7.0.0
+  - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
   - pytest-asyncio>=0.17.0

ci/deps/actions-39.yaml

@@ -11,7 +11,7 @@ dependencies:
   - meson-python=0.13.1
 
   # test dependencies
-  - pytest>=7.0.0
+  - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
   - pytest-asyncio>=0.17.0

ci/deps/actions-pypy-39.yaml

@@ -14,7 +14,7 @@ dependencies:
   - meson-python=0.13.1
 
   # test dependencies
-  - pytest>=7.0.0
+  - pytest>=7.3.2
   - pytest-cov
   - pytest-asyncio>=0.17.0
   - pytest-xdist>=2.2.0

ci/deps/circle-310-arm64.yaml

@@ -11,7 +11,7 @@ dependencies:
   - meson-python=0.13.1
 
   # test dependencies
-  - pytest>=7.0.0
+  - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
   - pytest-asyncio>=0.17.0

ci/meta.yaml

@@ -62,7 +62,7 @@ test:
     - python -c "import pandas; pandas.test(extra_args={{ extra_args }})"  # [python_impl == "cpython"]
   requires:
     - pip
-    - pytest >=7.0.0
+    - pytest >=7.3.2
     - pytest-asyncio >=0.17.0
     - pytest-xdist >=2.2.0
     - pytest-cov

doc/source/user_guide/io.rst

@@ -2111,7 +2111,8 @@ Reading from a JSON string:
 
 .. ipython:: python
 
-   pd.read_json(json)
+   from io import StringIO
+   pd.read_json(StringIO(json))
 
 Reading from a file:
 
@@ -2135,6 +2136,7 @@ Preserve string indices:
 
 .. ipython:: python
 
+   from io import StringIO
    si = pd.DataFrame(
        np.zeros((4, 4)), columns=list(range(4)), index=[str(i) for i in range(4)]
    )
@@ -2143,7 +2145,7 @@ Preserve string indices:
    si.columns
    json = si.to_json()
 
-   sij = pd.read_json(json, convert_axes=False)
+   sij = pd.read_json(StringIO(json), convert_axes=False)
    sij
    sij.index
    sij.columns
@@ -2152,18 +2154,19 @@ Dates written in nanoseconds need to be read back in nanoseconds:
 
 .. ipython:: python
 
+   from io import StringIO
    json = dfj2.to_json(date_unit="ns")
 
    # Try to parse timestamps as milliseconds -> Won't Work
-   dfju = pd.read_json(json, date_unit="ms")
+   dfju = pd.read_json(StringIO(json), date_unit="ms")
    dfju
 
    # Let pandas detect the correct precision
-   dfju = pd.read_json(json)
+   dfju = pd.read_json(StringIO(json))
    dfju
 
    # Or specify that all timestamps are in nanoseconds
-   dfju = pd.read_json(json, date_unit="ns")
+   dfju = pd.read_json(StringIO(json), date_unit="ns")
    dfju
 
 By setting the ``dtype_backend`` argument you can control the default dtypes used for the resulting DataFrame.
@@ -2251,11 +2254,12 @@ For line-delimited json files, pandas can also return an iterator which reads in
 
 .. ipython:: python
 
+  from io import StringIO
   jsonl = """
       {"a": 1, "b": 2}
       {"a": 3, "b": 4}
   """
-  df = pd.read_json(jsonl, lines=True)
+  df = pd.read_json(StringIO(jsonl), lines=True)
   df
   df.to_json(orient="records", lines=True)
 

doc/source/user_guide/timedeltas.rst

@@ -259,9 +259,6 @@ an alternative is to divide by another timedelta object. Note that division by t
    # to days
    td / np.timedelta64(1, "D")
 
-   # to months (these are constant months)
-   td / np.timedelta64(1, "M")
-
 Dividing or multiplying a ``timedelta64[ns]`` Series by an integer or integer Series
 yields another ``timedelta64[ns]`` dtypes Series.
 

doc/source/user_guide/window.rst

@@ -89,6 +89,7 @@ For example, a `weighted mean <https://en.wikipedia.org/wiki/Weighted_arithmetic
 be calculated with :meth:`~Rolling.apply` by specifying a separate column of weights.
 
 .. ipython:: python
+   :okwarning:
 
    def weighted_mean(x):
        arr = np.ones((1, x.shape[1]))
@@ -114,6 +115,7 @@ the ``update`` argument to continue the windowing calculation.
    df.ewm(0.5).mean()
 
 .. ipython:: python
+   :okwarning:
 
    online_ewm = df.head(2).ewm(0.5).online()
    online_ewm.mean()

doc/source/whatsnew/v1.5.0.rst

@@ -474,19 +474,22 @@ upon serialization. (Related issue :issue:`12997`)
 
 .. code-block:: ipython
 
-    In [4]: a.to_json(date_format='iso')
-    Out[4]: '{"2020-12-28T00:00:00.000Z":0,"2020-12-28T01:00:00.000Z":1,"2020-12-28T02:00:00.000Z":2}'
+    In [4]: from io import StringIO
 
-    In [5]: pd.read_json(a.to_json(date_format='iso'), typ="series").index == a.index
-    Out[5]: array([False, False, False])
+    In [5]: a.to_json(date_format='iso')
+    Out[5]: '{"2020-12-28T00:00:00.000Z":0,"2020-12-28T01:00:00.000Z":1,"2020-12-28T02:00:00.000Z":2}'
+
+    In [6]: pd.read_json(StringIO(a.to_json(date_format='iso')), typ="series").index == a.index
+    Out[6]: array([False, False, False])
 
 *New Behavior*
 
 .. ipython:: python
 
+    from io import StringIO
     a.to_json(date_format='iso')
     # Roundtripping now works
-    pd.read_json(a.to_json(date_format='iso'), typ="series").index == a.index
+    pd.read_json(StringIO(a.to_json(date_format='iso')), typ="series").index == a.index
 
 .. _whatsnew_150.notable_bug_fixes.groupby_value_counts_categorical:
 

doc/source/whatsnew/v2.0.3.rst

@@ -14,19 +14,21 @@ including other versions of pandas.
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
 - Fixed performance regression in merging on datetime-like columns (:issue:`53231`)
+- For external ExtensionArray implementations, restored the default use of ``_values_for_factorize`` for hashing arrays (:issue:`53475`)
 -
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_203.bug_fixes:
 
 Bug fixes
 ~~~~~~~~~
-- Bug in :class:`DateOffset` which had inconsistent behavior when multiplying a :class:`DateOffset` object by a constant (:issue:`47953`)
 - Bug in :func:`RangeIndex.union` when using ``sort=True`` with another :class:`RangeIndex` (:issue:`53490`)
+- Bug in :func:`Series.reindex` when expanding a non-nanosecond datetime or timedelta :class:`Series` would not fill with ``NaT`` correctly (:issue:`53497`)
 - Bug in :func:`read_csv` when defining ``dtype`` with ``bool[pyarrow]`` for the ``"c"`` and ``"python"`` engines (:issue:`53390`)
 - Bug in :meth:`Series.str.split` and :meth:`Series.str.rsplit` with ``expand=True`` for :class:`ArrowDtype` with ``pyarrow.string`` (:issue:`53532`)
 - Bug in indexing methods (e.g. :meth:`DataFrame.__getitem__`) where taking the entire :class:`DataFrame`/:class:`Series` would raise an ``OverflowError`` when Copy on Write was enabled and the length of the array was over the maximum size a 32-bit integer can hold (:issue:`53616`)
--
+- Bug when constructing a :class:`DataFrame` with columns of an :class:`ArrowDtype` with a ``pyarrow.dictionary`` type that reindexes the data (:issue:`53617`)
+- Bug when indexing a :class:`DataFrame` or :class:`Series` with an :class:`Index` with a timestamp :class:`ArrowDtype` would raise an ``AttributeError`` (:issue:`53644`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_203.other: