diff --git a/nibabel/tests/test_viewers.py b/nibabel/tests/test_viewers.py
index 53f4a32bd..72d839c92 100644
--- a/nibabel/tests/test_viewers.py
+++ b/nibabel/tests/test_viewers.py
@@ -102,3 +102,35 @@ def test_viewer():
     v2.link_to(v1)  # shouldn't do anything
     v1.close()
     v2.close()
+
+
+@needs_mpl
+def test_viewer_nonRAS():
+    data1 = np.random.rand(10, 20, 40)
+    data1[5, 10, :] = 0
+    data1[5, :, 30] = 0
+    data1[:, 10, 30] = 0
+    # RSA affine
+    aff1 = np.array([[1, 0, 0, -5], [0, 0, 1, -30], [0, 1, 0, -10], [0, 0, 0, 1]])
+    o1 = OrthoSlicer3D(data1, aff1)
+    sag = o1._ims[0].get_array()
+    cor = o1._ims[1].get_array()
+    axi = o1._ims[2].get_array()
+
+    # Sagittal view: [0, I->S, P->A], so data is transposed, matching plot array
+    assert_array_equal(sag, data1[5, :, :])
+    # Coronal view: [L->R, I->S, 0]. Data is not transposed, transpose to match plot array
+    assert_array_equal(cor, data1[:, :, 30].T)
+    # Axial view: [L->R, 0, P->A]. Data is not transposed, transpose to match plot array
+    assert_array_equal(axi, data1[:, 10, :].T)
+
+    o1.set_position(1, 2, 3)  # R, A, S coordinates
+
+    sag = o1._ims[0].get_array()
+    cor = o1._ims[1].get_array()
+    axi = o1._ims[2].get_array()
+
+    # Shift 1 right, 2 anterior, 3 superior
+    assert_array_equal(sag, data1[6, :, :])
+    assert_array_equal(cor, data1[:, :, 32].T)
+    assert_array_equal(axi, data1[:, 13, :].T)
diff --git a/nibabel/viewers.py b/nibabel/viewers.py
index 1e927544b..0dc2f0daf 100644
--- a/nibabel/viewers.py
+++ b/nibabel/viewers.py
@@ -399,7 +399,8 @@ def _set_position(self, x, y, z, notify=True):
         # deal with slicing appropriately
         self._position[:3] = [x, y, z]
         idxs = np.dot(self._inv_affine, self._position)[:3]
-        for ii, (size, idx) in enumerate(zip(self._sizes, idxs)):
+        idxs_new_order = idxs[self._order]
+        for ii, (size, idx) in enumerate(zip(self._sizes, idxs_new_order)):
             self._data_idx[ii] = max(min(int(round(idx)), size - 1), 0)
         for ii in range(3):
             # sagittal: get to S/A
@@ -491,10 +492,11 @@ def _on_mouse(self, event):
             x, y = event.xdata, event.ydata
             x = self._sizes[xax] - x if self._flips[xax] else x
             y = self._sizes[yax] - y if self._flips[yax] else y
-            idxs = [None, None, None, 1.0]
+            idxs = np.ones(4)
             idxs[xax] = x
             idxs[yax] = y
             idxs[ii] = self._data_idx[ii]
+            idxs[:3] = idxs[self._order]
             self._set_position(*np.dot(self._affine, idxs)[:3])
         self._draw()