PERF: speedup vectorized string functions by:

       push to cython, eliminate convert_objects call by creating directly
       (but need specific routines that have a call signature for this)
       overhead still in the actual x.endswith call (which operates on a pyobject)

Author: jreback

pandas/core/strings.py

@@ -6,7 +6,6 @@
 import re
 import pandas.lib as lib
 
-
 def _get_array_list(arr, others):
     if isinstance(others[0], (list, np.ndarray)):
         arrays = [arr] + list(others)
@@ -203,9 +202,26 @@ def str_endswith(arr, pat, na=np.nan):
     -------
     endswith : array (boolean)
     """
-    f = lambda x: x.endswith(pat)
-    return _na_map(f, arr, na)
-
+    if not isinstance(arr, np.ndarray):
+        arr = np.asarray(arr, dtype=object)
+    if True:
+        mask = isnull(arr)
+        try:
+            result = lib.string_na_map_bool(arr.values, 'endswith', [ pat ], mask.view(np.uint8))
+        except (TypeError, AttributeError):
+            def g(x):
+                try:
+                    return f(x)
+                except (TypeError, AttributeError):
+                    return na_value
+            return _map(g, arr)
+        #if na_value is not np.nan:
+        #    np.putmask(result, mask, na_value)
+        #    if result.dtype == object:
+        #        result = lib.maybe_convert_objects(result)
+        return result
+    else:
+        return lib.map_infer(arr, f)
 
 def str_lower(arr):
     """

pandas/lib.pyx

@@ -1280,3 +1280,4 @@ def indices_fast(object index, ndarray[int64_t] labels, list keys,
 include "reduce.pyx"
 include "properties.pyx"
 include "inference.pyx"
+include "strings.pyx"

pandas/src/strings.pyx

@@ -0,0 +1,34 @@
+cimport util
+
+def string_na_map_bool(ndarray arr, object fn, list args, ndarray[uint8_t] mask,
+                   bint convert=1):
+    '''
+    Substitute for np.vectorize with pandas-friendly dtype inference
+
+    Parameters
+    ----------
+    arr : ndarray
+    fm : function name
+    args : a list of the args to pass function
+
+    Returns
+    -------
+    mapped : ndarray
+    '''
+    cdef:
+        Py_ssize_t i, n
+        ndarray[uint8_t] result
+
+    n = len(arr)
+    result = np.empty(n, dtype=np.uint8)
+    for i in range(n):
+        if mask[i]:
+            result[i] = 0
+        else:
+            result[i] = string_endswith(arr[i], args[0])
+
+    return result
+
+cdef inline bint string_endswith(char *x, object pat):
+     return x.endswith(pat)
+

setup.py

@@ -561,7 +561,7 @@ def run(self):
     cmdclass['build_src'] = DummyBuildSrc
     cmdclass['build_ext'] = CheckingBuildExt
 
-lib_depends = ['reduce', 'inference', 'properties']
+lib_depends = ['reduce', 'inference', 'properties', 'strings']
 
 
 def srcpath(name=None, suffix='.pyx', subdir='src'):