diff --git a/sklearn/utils/tests/test_utils.py b/sklearn/utils/tests/test_utils.py
index dae08d0086fde156b8e45782018c69d5c8c26053..441f5d04dc500724edd5af5e0a7472e4995012c4 100644
--- a/sklearn/utils/tests/test_utils.py
+++ b/sklearn/utils/tests/test_utils.py
@@ -93,7 +93,27 @@ def test_safe_mask():
     assert_equal(X_csr[mask].shape[0], 3)
 
 
-def test_pinvh():
-    a = np.random.randn(5, 3)
-    a = np.dot(a, a.T)  # symmetric singular matrix
-    assert_almost_equal(pinv2(a), pinvh(a))
+def test_pinvh_simple_real():
+    a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 10]], dtype=np.float64)
+    a = np.dot(a, a.T)
+    a_pinv = pinvh(a)
+    assert_almost_equal(np.dot(a, a_pinv), np.eye(3))
+
+
+def test_pinvh_nonpositive():
+    a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float64)
+    a = np.dot(a, a.T)
+    u, s, vt = np.linalg.svd(a)
+    s[0] *= -1
+    a = np.dot(u * s, vt)  # a is now symmetric non-positive and singular
+    a_pinv = pinv2(a)
+    a_pinvh = pinvh(a)
+    assert_almost_equal(a_pinv, a_pinvh)
+
+
+def test_pinvh_simple_complex():
+    a = (np.array([[1, 2, 3], [4, 5, 6], [7, 8, 10]])
+         + 1j * np.array([[10, 8, 7], [6, 5, 4], [3, 2, 1]]))
+    a = np.dot(a, a.conj().T)
+    a_pinv = pinvh(a)
+    assert_almost_equal(np.dot(a, a_pinv), np.eye(3))