From 38c2251d84a7792d90fc7eb65aca775da05eb7bd Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Sat, 24 Oct 2015 11:26:03 -0700 Subject: [PATCH 1/6] figure out appveyor issue --- nibabel/tests/test_round_trip.py | 44 ++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/nibabel/tests/test_round_trip.py b/nibabel/tests/test_round_trip.py index 52c29c98cf..9f61e98186 100644 --- a/nibabel/tests/test_round_trip.py +++ b/nibabel/tests/test_round_trip.py @@ -17,6 +17,7 @@ DEBUG = True + def round_trip(arr, out_dtype): img = Nifti1Image(arr, np.eye(4)) img.file_map['image'].fileobj = BytesIO() @@ -89,6 +90,7 @@ def test_big_bad_ulp(): BIG_FLOAT = np.float64 + def test_round_trip(): scaling_type = np.float32 rng = np.random.RandomState(20111121) @@ -137,7 +139,7 @@ def check_arr(test_id, V_in, in_type, out_type, scaling_type): return rel_err = np.abs(top / arr) abs_err = np.abs(top) - if slope == 1: # integers output, offset only scaling + if slope == 1: # integers output, offset only scaling if set((in_type, out_type)) == set((np.int64, np.uint64)): # Scaling to or from 64 bit ints can go outside range of continuous # integers for float64 and thus lose precision; take this into @@ -167,28 +169,32 @@ def check_arr(test_id, V_in, in_type, out_type, scaling_type): rel_thresh = ulp(scaling_type(1)) test_vals = (abs_err <= exp_abs_err) | (rel_err <= rel_thresh) this_test = np.all(test_vals) - if DEBUG: + if not this_test and DEBUG: abs_fails = (abs_err > exp_abs_err) rel_fails = (rel_err > rel_thresh) all_fails = abs_fails & rel_fails + + if np.any(abs_fails): + abs_max_diff = -(abs_err - exp_abs_err)[abs_fails].max() + abs_mx_e = abs_err[abs_fails].max() + exp_abs_mx_e = exp_abs_err[abs_fails].max() if np.any(rel_fails): - abs_mx_e = abs_err[rel_fails].max() - exp_abs_mx_e = exp_abs_err[rel_fails].max() - else: - abs_mx_e = None - exp_abs_mx_e = None + rel_max_diff = -(rel_err - rel_thresh)[rel_fails].max() + rel_mx_e = rel_err[rel_fails].max() + + print("Test ID: %s; in_type=%s, out_type=%s""" % ( + test_id, np.dtype(in_type).str, np.dtype(out_type).str)) + print("\tslope=%.5e, inter=%.5e" % (slope, inter)) + if np.any(abs_fails): - rel_mx_e = rel_err[abs_fails].max() - else: - rel_mx_e = None - print (test_id, - np.dtype(in_type).str, - np.dtype(out_type).str, - exp_abs_mx_e, - abs_mx_e, - rel_thresh, - rel_mx_e, - slope, inter) + print("\tABS FAIL: exp_abs_mx_e=%.5e < abs_mx_e=%.5e; max_diff=%.5e" % ( + exp_abs_mx_e, abs_mx_e, abs_max_diff)) + if np.any(rel_fails) is not None: + print("\tREL FAIL: rel_thresh =%.5e < rel_mx_e=%.5e; max_diff=%.5e" % ( + rel_thresh, rel_mx_e, rel_max_diff)) + print("") + # To help debugging failures with --pdb-failure fail_i = np.nonzero(all_fails) - assert_true(this_test) + assert_true(this_test, "types == %s, %s; see stdout for details" % ( + in_type, out_type)) From 3d6e7f6bdec3087966890e65d2e5423d3e443dfb Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 28 Oct 2015 12:40:10 -0400 Subject: [PATCH 2/6] Add error frequency information --- nibabel/tests/test_round_trip.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/nibabel/tests/test_round_trip.py b/nibabel/tests/test_round_trip.py index 9f61e98186..0527b20fe7 100644 --- a/nibabel/tests/test_round_trip.py +++ b/nibabel/tests/test_round_trip.py @@ -174,24 +174,27 @@ def check_arr(test_id, V_in, in_type, out_type, scaling_type): rel_fails = (rel_err > rel_thresh) all_fails = abs_fails & rel_fails - if np.any(abs_fails): - abs_max_diff = -(abs_err - exp_abs_err)[abs_fails].max() - abs_mx_e = abs_err[abs_fails].max() - exp_abs_mx_e = exp_abs_err[abs_fails].max() - if np.any(rel_fails): - rel_max_diff = -(rel_err - rel_thresh)[rel_fails].max() - rel_mx_e = rel_err[rel_fails].max() - print("Test ID: %s; in_type=%s, out_type=%s""" % ( test_id, np.dtype(in_type).str, np.dtype(out_type).str)) print("\tslope=%.5e, inter=%.5e" % (slope, inter)) if np.any(abs_fails): - print("\tABS FAIL: exp_abs_mx_e=%.5e < abs_mx_e=%.5e; max_diff=%.5e" % ( - exp_abs_mx_e, abs_mx_e, abs_max_diff)) - if np.any(rel_fails) is not None: - print("\tREL FAIL: rel_thresh =%.5e < rel_mx_e=%.5e; max_diff=%.5e" % ( - rel_thresh, rel_mx_e, rel_max_diff)) + abs_max_diff = (abs_err - exp_abs_err)[abs_fails].max() + abs_mx_e = abs_err[abs_fails].max() + exp_abs_mx_e = exp_abs_err[abs_fails].max() + print("\tABS FAIL: exp_abs_mx_e=%.5e < abs_mx_e=%.5e; " + "max_diff=%.5e; num=%d/%d" % (exp_abs_mx_e, abs_mx_e, + abs_max_diff, + np.sum(abs_fails), + np.product(abs_fails.shape))) + if np.any(rel_fails): + rel_max_diff = (rel_err - rel_thresh)[rel_fails].max() + rel_mx_e = rel_err[rel_fails].max() + print("\tREL FAIL: rel_thresh =%.5e < rel_mx_e=%.5e; " + "max_diff=%.5e; num=%d/%d" % (rel_thresh, rel_mx_e, + rel_max_diff, + np.sum(rel_fails), + np.product(rel_fails.shape))) print("") # To help debugging failures with --pdb-failure From 4fd1682d0462233f1b78760ff74d1510c6c20073 Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Thu, 5 Nov 2015 05:04:52 -0800 Subject: [PATCH 3/6] STY: minor code cleanup. --- nibabel/tests/test_round_trip.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/nibabel/tests/test_round_trip.py b/nibabel/tests/test_round_trip.py index 0527b20fe7..090ef7595b 100644 --- a/nibabel/tests/test_round_trip.py +++ b/nibabel/tests/test_round_trip.py @@ -15,8 +15,6 @@ from numpy.testing import assert_array_equal, assert_almost_equal -DEBUG = True - def round_trip(arr, out_dtype): img = Nifti1Image(arr, np.eye(4)) @@ -127,11 +125,10 @@ def check_arr(test_id, V_in, in_type, out_type, scaling_type): if arr_dash is None: # Scaling causes a header or writer error return - nzs = arr != 0 # avoid divide by zero error + nzs = arr != 0 # avoid divide by zero error if not np.any(nzs): - if DEBUG: - raise ValueError('Array all zero') - return + raise ValueError('Array all zero') + arr = arr[nzs] arr_dash_L = arr_dash.astype(BIG_FLOAT)[nzs] top = arr - arr_dash_L @@ -139,6 +136,7 @@ def check_arr(test_id, V_in, in_type, out_type, scaling_type): return rel_err = np.abs(top / arr) abs_err = np.abs(top) + if slope == 1: # integers output, offset only scaling if set((in_type, out_type)) == set((np.int64, np.uint64)): # Scaling to or from 64 bit ints can go outside range of continuous @@ -148,9 +146,10 @@ def check_arr(test_id, V_in, in_type, out_type, scaling_type): Ai = A - inter ulps = [big_bad_ulp(A), big_bad_ulp(Ai)] exp_abs_err = np.max(ulps, axis=0) - else: # floats can give full precision - no error! + else: # floats can give full precision - no error! exp_abs_err = np.zeros_like(abs_err) rel_thresh = 0 + else: # Error from integer rounding inting_err = np.abs(scaling_type(slope) / 2) @@ -167,14 +166,15 @@ def check_arr(test_id, V_in, in_type, out_type, scaling_type): # This threshold needs to be 2 x larger on windows 32 bit and PPC for # some reason rel_thresh = ulp(scaling_type(1)) + test_vals = (abs_err <= exp_abs_err) | (rel_err <= rel_thresh) this_test = np.all(test_vals) - if not this_test and DEBUG: + + if not this_test: abs_fails = (abs_err > exp_abs_err) rel_fails = (rel_err > rel_thresh) - all_fails = abs_fails & rel_fails - print("Test ID: %s; in_type=%s, out_type=%s""" % ( + print("Test ID: %s; in_type=%s, out_type=%s" % ( test_id, np.dtype(in_type).str, np.dtype(out_type).str)) print("\tslope=%.5e, inter=%.5e" % (slope, inter)) @@ -185,19 +185,21 @@ def check_arr(test_id, V_in, in_type, out_type, scaling_type): print("\tABS FAIL: exp_abs_mx_e=%.5e < abs_mx_e=%.5e; " "max_diff=%.5e; num=%d/%d" % (exp_abs_mx_e, abs_mx_e, abs_max_diff, - np.sum(abs_fails), - np.product(abs_fails.shape))) + abs_fails.sum(), + abs_fails.size)) if np.any(rel_fails): rel_max_diff = (rel_err - rel_thresh)[rel_fails].max() rel_mx_e = rel_err[rel_fails].max() print("\tREL FAIL: rel_thresh =%.5e < rel_mx_e=%.5e; " "max_diff=%.5e; num=%d/%d" % (rel_thresh, rel_mx_e, rel_max_diff, - np.sum(rel_fails), - np.product(rel_fails.shape))) + rel_fails.sum(), + rel_fails.size)) print("") # To help debugging failures with --pdb-failure + all_fails = np.logical_or(abs_fails, rel_fails) fail_i = np.nonzero(all_fails) + assert_true(this_test, "types == %s, %s; see stdout for details" % ( in_type, out_type)) From 556c6f8a149e9d4eefb741b5a8ebe7ed89223b96 Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Thu, 5 Nov 2015 05:05:45 -0800 Subject: [PATCH 4/6] RF: put bad parameter checks into check_params. --- nibabel/tests/test_round_trip.py | 35 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/nibabel/tests/test_round_trip.py b/nibabel/tests/test_round_trip.py index 090ef7595b..70d4d7d9dd 100644 --- a/nibabel/tests/test_round_trip.py +++ b/nibabel/tests/test_round_trip.py @@ -11,8 +11,7 @@ from ..arraywriters import ScalingError from ..casting import best_float, ulp, type_info -from nose.tools import assert_true - +from nose.tools import assert_true, assert_false from numpy.testing import assert_array_equal, assert_almost_equal @@ -32,10 +31,17 @@ def check_params(in_arr, in_type, out_type): if arr.dtype.kind == 'f': info = np.finfo(in_type) arr = np.clip(arr, info.min, info.max) - try: - arr_dash, slope, inter = round_trip(arr, out_type) - except (ScalingError, HeaderDataError): - return arr, None, None, None + + arr_dash, slope, inter = round_trip(arr, out_type) + + assert_false(arr_dash is None, "Scaling causes a header or writer error") + + nzs = arr != 0 # avoid divide by zero error + assert_true(np.any(nzs), 'Array all zero') + + arr = arr[nzs] + arr_dash = arr_dash[nzs] + return arr, arr_dash, slope, inter @@ -121,19 +127,14 @@ def test_round_trip(): def check_arr(test_id, V_in, in_type, out_type, scaling_type): - arr, arr_dash, slope, inter = check_params(V_in, in_type, out_type) - if arr_dash is None: - # Scaling causes a header or writer error + try: + arr, arr_dash, slope, inter = check_params(V_in, in_type, out_type) + except (ScalingError, HeaderDataError): + # We assume this is reasonable; no more checks to do. return - nzs = arr != 0 # avoid divide by zero error - if not np.any(nzs): - raise ValueError('Array all zero') - - arr = arr[nzs] - arr_dash_L = arr_dash.astype(BIG_FLOAT)[nzs] + arr_dash_L = arr_dash.astype(BIG_FLOAT) top = arr - arr_dash_L - if not np.any(top != 0): - return + rel_err = np.abs(top / arr) abs_err = np.abs(top) From 25db6870c55c09d69d5101c22955ad19ea77fbf0 Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Thu, 5 Nov 2015 05:22:05 -0800 Subject: [PATCH 5/6] TST: Print extra test failure info. --- nibabel/tests/test_round_trip.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nibabel/tests/test_round_trip.py b/nibabel/tests/test_round_trip.py index 70d4d7d9dd..773199ec68 100644 --- a/nibabel/tests/test_round_trip.py +++ b/nibabel/tests/test_round_trip.py @@ -188,6 +188,7 @@ def check_arr(test_id, V_in, in_type, out_type, scaling_type): abs_max_diff, abs_fails.sum(), abs_fails.size)) + if np.any(rel_fails): rel_max_diff = (rel_err - rel_thresh)[rel_fails].max() rel_mx_e = rel_err[rel_fails].max() @@ -196,10 +197,15 @@ def check_arr(test_id, V_in, in_type, out_type, scaling_type): rel_max_diff, rel_fails.sum(), rel_fails.size)) + # Print up to two entries of raw and round trip data. + all_fails = np.logical_and(abs_fails, rel_fails) + print("\ttop[fails][:2] = %s" % str(top[all_fails][:2])) + print("\tarr[fails][:2] = %s" % str(arr[all_fails][:2])) + print("\tarr_dash[fails][:2] = %s" % str(arr_dash[all_fails][:2])) + print("\tarr_dash_L[fails][:2] = %s" % str(arr_dash_L[all_fails][:2])) print("") # To help debugging failures with --pdb-failure - all_fails = np.logical_or(abs_fails, rel_fails) fail_i = np.nonzero(all_fails) assert_true(this_test, "types == %s, %s; see stdout for details" % ( From ff8c46ebc4da01f1d8eed399c4be47bbf085b6ac Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Thu, 5 Nov 2015 09:08:45 -0800 Subject: [PATCH 6/6] Relative threshhold should not be zero! --- nibabel/tests/test_round_trip.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nibabel/tests/test_round_trip.py b/nibabel/tests/test_round_trip.py index 773199ec68..d59f5d5b03 100644 --- a/nibabel/tests/test_round_trip.py +++ b/nibabel/tests/test_round_trip.py @@ -147,9 +147,11 @@ def check_arr(test_id, V_in, in_type, out_type, scaling_type): Ai = A - inter ulps = [big_bad_ulp(A), big_bad_ulp(Ai)] exp_abs_err = np.max(ulps, axis=0) + rel_thresh = ulp(scaling_type(inter)) + else: # floats can give full precision - no error! exp_abs_err = np.zeros_like(abs_err) - rel_thresh = 0 + rel_thresh = 0 else: # Error from integer rounding