BUG: GH3317 Fix frequency inference to be automatic in a DatetimeIndex if freq is not specified during _ensure_index

PERF: remove automatic downcasting, instead do on-demand via 'downcast=infer'

TST: fix up failing tests

CLN: reindex/fill in internals moved to core/internals/Block/reindex_items_from

PERF: don't automatically downcast with a float block

BUG: GH3317 reverse prior fix in tseries/offsets, to change slightly the multi reindex

TST: identical index tests

BUG: GH4618 provide automatic downcasting on a reindexed with method Series
     (in this case a shifted boolean then filled series)

Author: jreback

doc/source/release.rst

@@ -115,6 +115,8 @@ pandas 0.13
     - ``MultiIndex.astype()`` now only allows ``np.object_``-like dtypes and
       now returns a ``MultiIndex`` rather than an ``Index``. (:issue:`4039`)
 
+  - Infer and downcast dtype if ``downcast='infer'`` is passed to ``fillna/ffill/bfill`` (:issue:`4604`)
+
 **Internal Refactoring**
 
 In 0.13.0 there is a major refactor primarily to subclass ``Series`` from ``NDFrame``,
@@ -183,11 +185,9 @@ See :ref:`Internal Refactoring<whatsnew_0130.refactoring>`
 
 - Indexing with dtype conversions fixed (:issue:`4463`, :issue:`4204`)
 
-- Refactor Series.reindex to core/generic.py (:issue:`4604`), allow ``method=`` in reindexing
+- Refactor Series.reindex to core/generic.py (:issue:`4604`, :issue:`4618`), allow ``method=`` in reindexing
   on a Series to work
 
-- Infer and downcast dtype if appropriate on ``ffill/bfill`` (:issue:`4604`)
-
 **Experimental Features**
 
 **Bug Fixes**
@@ -259,6 +259,8 @@ See :ref:`Internal Refactoring<whatsnew_0130.refactoring>`
   - Fix bug in ``pd.read_clipboard`` on windows with PY3 (:issue:`4561`); not decoding properly
   - ``tslib.get_period_field()`` and ``tslib.get_period_field_arr()`` now raise
     if code argument out of range (:issue:`4519`, :issue:`4520`)
+  - Fix reindexing with multiple axes; if an axes match was not replacing the current axes, leading
+    to a possible lazay frequency inference issue (:issue:`3317`)
 
 pandas 0.12
 ===========

doc/source/v0.13.0.txt

@@ -96,6 +96,8 @@ API changes
         # and all methods take an inplace kwarg
         index.set_names(["bob", "cranberry"], inplace=True)
 
+  - Infer and downcast dtype if ``downcast='infer'`` is passed to ``fillna/ffill/bfill`` (:issue:`4604`)
+
 Enhancements
 ~~~~~~~~~~~~
 
@@ -237,11 +239,9 @@ and behaviors. Series formerly subclassed directly from ``ndarray``. (:issue:`40
 
 - Indexing with dtype conversions fixed (:issue:`4463`, :issue:`4204`)
 
-- Refactor Series.reindex to core/generic.py (:issue:`4604`), allow ``method=`` in reindexing
+- Refactor Series.reindex to core/generic.py (:issue:`4604`, :issue:`4618`), allow ``method=`` in reindexing
   on a Series to work
 
-- Infer and downcast dtype if appropriate on ``ffill/bfill`` (:issue:`4604`)
-
 Bug Fixes
 ~~~~~~~~~
 

pandas/core/common.py

@@ -990,14 +990,20 @@ def _possibly_downcast_to_dtype(result, dtype):
         if issubclass(dtype.type, np.floating):
             return result.astype(dtype)
         elif dtype == np.bool_ or issubclass(dtype.type, np.integer):
+
+            # do a test on the first element, if it fails then we are done
+            r = result.ravel()
+            arr = np.array([ r[0] ])
+            if (arr != arr.astype(dtype)).item():
+                return result
+
+            # a comparable, e.g. a Decimal may slip in here
+            elif not isinstance(r[0], (np.integer,np.floating,np.bool,int,float,bool)):
+                return result
+
             if issubclass(result.dtype.type, (np.object_,np.number)) and notnull(result).all():
                 new_result = result.astype(dtype)
                 if (new_result == result).all():
-
-                    # a comparable, e.g. a Decimal may slip in here
-                    if not isinstance(result.ravel()[0], (np.integer,np.floating,np.bool,int,float,bool)):
-                        return result
-
                     return new_result
     except:
         pass
@@ -1174,7 +1180,7 @@ def backfill_2d(values, limit=None, mask=None):
         pass
     return values
 
-def interpolate_2d(values, method='pad', axis=0, limit=None, missing=None):
+def interpolate_2d(values, method='pad', axis=0, limit=None, fill_value=None):
     """ perform an actual interpolation of values, values will be make 2-d if needed
         fills inplace, returns the result """
 
@@ -1187,10 +1193,10 @@ def interpolate_2d(values, method='pad', axis=0, limit=None, missing=None):
             raise Exception("cannot interpolate on a ndim == 1 with axis != 0")
         values = values.reshape(tuple((1,) + values.shape))
 
-    if missing is None:
+    if fill_value is None:
         mask = None
     else:  # todo create faster fill func without masking
-        mask = mask_missing(transf(values), missing)
+        mask = mask_missing(transf(values), fill_value)
 
     method = _clean_fill_method(method)
     if method == 'pad':
@@ -1870,6 +1876,7 @@ def _astype_nansafe(arr, dtype, copy=True):
 
 
 def _clean_fill_method(method):
+    if method is None: return None
     method = method.lower()
     if method == 'ffill':
         method = 'pad'

pandas/core/frame.py

@@ -2280,12 +2280,9 @@ def _reindex_multi(self, axes, copy, fill_value):
                                            fill_value=fill_value)
             return self._constructor(new_values, index=new_index,
                                      columns=new_columns)
-        elif row_indexer is not None:
-            return self._reindex_with_indexers({0: [new_index,   row_indexer]}, copy=copy, fill_value=fill_value)
-        elif col_indexer is not None:
-            return self._reindex_with_indexers({1: [new_columns, col_indexer]}, copy=copy, fill_value=fill_value)
         else:
-            return self.copy() if copy else self
+            return self._reindex_with_indexers({0: [new_index,   row_indexer],
+                                                1: [new_columns, col_indexer]}, copy=copy, fill_value=fill_value)
 
     def reindex_like(self, other, method=None, copy=True, limit=None,
                      fill_value=NA):

pandas/core/generic.py

@@ -987,7 +987,7 @@ def reindex(self, *args, **kwargs):
 
         # construct the args
         axes, kwargs = self._construct_axes_from_arguments(args, kwargs)
-        method = kwargs.get('method')
+        method = com._clean_fill_method(kwargs.get('method'))
         level = kwargs.get('level')
         copy = kwargs.get('copy', True)
         limit = kwargs.get('limit')
@@ -1082,6 +1082,7 @@ def reindex_axis(self, labels, axis=0, method=None, level=None, copy=True,
 
         axis_name = self._get_axis_name(axis)
         axis_values = self._get_axis(axis_name)
+        method = com._clean_fill_method(method)
         new_index, indexer = axis_values.reindex(labels, method, level,
                                                  limit=limit, copy_if_needed=True)
         return self._reindex_with_indexers({axis: [new_index, indexer]}, method=method, fill_value=fill_value,
@@ -1102,7 +1103,7 @@ def _reindex_with_indexers(self, reindexers, method=None, fill_value=np.nan, lim
             # reindex the axis
             if method is not None:
                 new_data = new_data.reindex_axis(
-                    index, method=method, axis=baxis,
+                    index, indexer=indexer, method=method, axis=baxis,
                     fill_value=fill_value, limit=limit, copy=copy)
 
             elif indexer is not None:
@@ -1419,7 +1420,8 @@ def fillna(self, value=None, method=None, axis=0, inplace=False,
         limit : int, default None
             Maximum size gap to forward or backward fill
         downcast : dict, default is None, a dict of item->dtype of what to
-            downcast if possible
+            downcast if possible, or the string 'infer' which will try to
+            downcast to an appropriate equal type (e.g. float64 to int64 if possible)
 
         See also
         --------
@@ -1438,6 +1440,7 @@ def fillna(self, value=None, method=None, axis=0, inplace=False,
         if axis + 1 > self._AXIS_LEN:
             raise ValueError(
                 "invalid axis passed for object type {0}".format(type(self)))
+        method = com._clean_fill_method(method)
 
         if value is None:
             if method is None:
@@ -1488,13 +1491,13 @@ def fillna(self, value=None, method=None, axis=0, inplace=False,
         else:
             return self._constructor(new_data)
 
-    def ffill(self, axis=0, inplace=False, limit=None):
+    def ffill(self, axis=0, inplace=False, limit=None, downcast=None):
         return self.fillna(method='ffill', axis=axis, inplace=inplace,
-                           limit=limit, downcast='infer')
+                           limit=limit, downcast=downcast)
 
-    def bfill(self, axis=0, inplace=False, limit=None):
+    def bfill(self, axis=0, inplace=False, limit=None, downcast=None):
         return self.fillna(method='bfill', axis=axis, inplace=inplace,
-                           limit=limit, downcast='infer')
+                           limit=limit, downcast=downcast)
 
     def replace(self, to_replace=None, value=None, inplace=False, limit=None,
                 regex=False, method=None, axis=None):
@@ -2046,6 +2049,7 @@ def align(self, other, join='outer', axis=None, level=None, copy=True,
             Aligned objects
         """
         from pandas import DataFrame, Series
+        method = com._clean_fill_method(method)
 
         if isinstance(other, DataFrame):
             return self._align_frame(other, join=join, axis=axis, level=level,

pandas/core/index.py

@@ -551,7 +551,7 @@ def identical(self, other):
         Similar to equals, but check that other comparable attributes are also equal
         """
         return self.equals(other) and all(
-            [ getattr(self,c,None) == getattr(other,c,None) for c in self._comparables ])
+            ( getattr(self,c,None) == getattr(other,c,None) for c in self._comparables ))
 
     def asof(self, label):
         """
@@ -1555,6 +1555,7 @@ class MultiIndex(Index):
     _names = FrozenList()
     _levels = FrozenList()
     _labels = FrozenList()
+    _comparables = ['names']
 
     def __new__(cls, levels=None, labels=None, sortorder=None, names=None,
                 copy=False):