Cythonized GroupBy mad

doc/source/whatsnew/v0.23.0.txt

@@ -791,6 +791,7 @@ Performance Improvements
 - Improved performance of variable ``.rolling()`` on ``.min()`` and ``.max()`` (:issue:`19521`)
 - Improved performance of :func:`pandas.core.groupby.GroupBy.ffill` and :func:`pandas.core.groupby.GroupBy.bfill` (:issue:`11296`)
 - Improved performance of :func:`pandas.core.groupby.GroupBy.any` and :func:`pandas.core.groupby.GroupBy.all` (:issue:`15435`)
+- Improved performance of :func:`pandas.core.groupby.GroupBy.mad` (:issue:`19165`)
 
 .. _whatsnew_0230.docs:
 

pandas/core/groupby.py

@@ -1353,6 +1353,26 @@ def var(self, ddof=1, *args, **kwargs):
             f = lambda x: x.var(ddof=ddof, **kwargs)
             return self._python_agg_general(f)
 
+    @Substitution(name='groupby')
+    @Appender(_doc_template)
+    def mad(self, skipna=True):
+        if not skipna:
+            raise NotImplementedError("'skipna=False' not yet implemented")
+
+        if self.axis != 0:
+            return self.apply(lambda x: x.mad(axis=self.axis))
+
+        # Wrap in a try..except to catch a TypeError with bool data
+        # Ideally this would be implemented in `mean` instead of here
+        try:
+            demeaned = np.abs(self.shift(0) - self.transform('mean'))
+            result = demeaned.groupby(self.grouper.labels).mean()
+            result.index = self.grouper.result_index
+        except TypeError:
+            raise DataError('No numeric types to aggregate')
+
+        return result
+
     @Substitution(name='groupby')
     @Appender(_doc_template)
     def sem(self, ddof=1):

pandas/tests/groupby/test_groupby.py

@@ -9,6 +9,7 @@
 from pandas import (date_range, bdate_range, Timestamp,
                     Index, MultiIndex, DataFrame, Series,
                     concat, Panel, DatetimeIndex, read_csv)
+from pandas.core.base import DataError
 from pandas.core.dtypes.missing import isna
 from pandas.errors import UnsupportedFunctionCall, PerformanceWarning
 from pandas.util.testing import (assert_frame_equal, assert_index_equal,
@@ -1300,17 +1301,6 @@ def test_non_cython_api(self):
         g = df.groupby('A')
         gni = df.groupby('A', as_index=False)
 
-        # mad
-        expected = DataFrame([[0], [np.nan]], columns=['B'], index=[1, 3])
-        expected.index.name = 'A'
-        result = g.mad()
-        assert_frame_equal(result, expected)
-
-        expected = DataFrame([[0., 0.], [0, np.nan]], columns=['A', 'B'],
-                             index=[0, 1])
-        result = gni.mad()
-        assert_frame_equal(result, expected)
-
         # describe
         expected_index = pd.Index([1, 3], name='A')
         expected_col = pd.MultiIndex(levels=[['B'],
@@ -2141,6 +2131,52 @@ def test_groupby_bool_aggs(self, agg_func, skipna, vals):
         result = getattr(df.groupby('key'), agg_func)(skipna=skipna)
         assert_frame_equal(result, exp_df)
 
+    @pytest.mark.parametrize("klass", [Series, DataFrame])
+    @pytest.mark.parametrize("test_mi", [True, False])
+    @pytest.mark.parametrize("dtype", ['int', 'float'])
+    def test_groupby_mad(self, klass, test_mi, dtype):
+        vals = np.array(range(10)).astype(dtype)
+        df = DataFrame({'key': ['a'] * 5 + ['b'] * 5, 'val': vals})
+
+        idx = pd.Index(['a', 'b'], name='key')
+        exp = klass([1.2, 1.2], index=idx)
+        grping = ['key']
+
+        if test_mi:
+            df = df.append(df)  # Double the size of the frame
+            df['newcol'] = ['foo'] * 10 + ['bar'] * 10
+            grping.append('newcol')
+
+            mi = pd.MultiIndex.from_product((exp.index.values,
+                                             ['bar', 'foo']),
+                                            names=['key', 'newcol'])
+            exp = exp.append(exp)
+            exp.index = mi
+
+        if klass is Series:
+            exp.name = 'val'
+            result = df.groupby(grping)['val'].mad()
+            tm.assert_series_equal(result, exp)
+        else:
+            exp = exp.rename(columns={0: 'val'})
+            result = df.groupby(grping).mad()
+            tm.assert_frame_equal(result, exp)
+
+    @pytest.mark.parametrize("vals", [
+        ['foo'] * 10, [True] * 10])
+    def test_groupby_mad_raises(self, vals):
+        df = DataFrame({'key': ['a'] * 5 + ['b'] * 5, 'val': vals})
+
+        with tm.assert_raises_regex(DataError,
+                                    "No numeric types to aggregate"):
+            df.groupby('key').mad()
+
+    def test_groupby_mad_skipna(self):
+        df = DataFrame({'key': ['a'] * 5 + ['b'] * 5, 'val': range(10)})
+        with tm.assert_raises_regex(
+                NotImplementedError, "'skipna=False' not yet implemented"):
+            df.groupby('key').mad(skipna=False)
+
     def test_dont_clobber_name_column(self):
         df = DataFrame({'key': ['a', 'a', 'a', 'b', 'b', 'b'],
                         'name': ['foo', 'bar', 'baz'] * 2})

pandas/tests/groupby/test_whitelist.py

@@ -12,7 +12,7 @@
 
 AGG_FUNCTIONS = ['sum', 'prod', 'min', 'max', 'median', 'mean', 'skew',
                  'mad', 'std', 'var', 'sem']
-AGG_FUNCTIONS_WITH_SKIPNA = ['skew', 'mad']
+AGG_FUNCTIONS_WITH_SKIPNA = ['skew']
 
 df_whitelist = frozenset([
     'last',