diff --git a/doc/whats_new.rst b/doc/whats_new.rst
index 492a0647ee31130ec45bd62cb7e1dd17c9a8a5d8..dfc2be923eff29e72128f1beee77b8af5cf0da65 100644
--- a/doc/whats_new.rst
+++ b/doc/whats_new.rst
@@ -292,6 +292,8 @@ Bug fixes
      when the length of features_names does not match n_features in the decision
      tree.
      :issue:`8512` by :user:`Li Li <aikinogard>`.
+   - Fixed a bug in :class:`manifold.TSNE` affecting convergence of the
+     gradient descent. :issue:`8768` by :user:`David DeTomaso <deto>`.
 
 API changes summary
 -------------------
diff --git a/sklearn/manifold/t_sne.py b/sklearn/manifold/t_sne.py
index b31f34d9eef13a74f1597a245ce7c59e96500626..30b9ca16e88c2f7611902e706791f84eaad68337 100644
--- a/sklearn/manifold/t_sne.py
+++ b/sklearn/manifold/t_sne.py
@@ -388,11 +388,11 @@ def _gradient_descent(objective, p0, it, n_iter, objective_error=None,
         new_error, grad = objective(p, *args, **kwargs)
         grad_norm = linalg.norm(grad)
 
-        inc = update * grad >= 0.0
+        inc = update * grad < 0.0
         dec = np.invert(inc)
-        gains[inc] += 0.05
-        gains[dec] *= 0.95
-        np.clip(gains, min_gain, np.inf)
+        gains[inc] += 0.2
+        gains[dec] *= 0.8
+        np.clip(gains, min_gain, np.inf, out=gains)
         grad *= gains
         update = momentum * update - learning_rate * grad
         p += update
@@ -631,10 +631,10 @@ class TSNE(BaseEstimator):
     >>> model = TSNE(n_components=2, random_state=0)
     >>> np.set_printoptions(suppress=True)
     >>> model.fit_transform(X) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
-    array([[ 0.00017599,  0.00003993],
-           [ 0.00009891,  0.00021913],
-           [ 0.00018554, -0.00009357],
-           [ 0.00009528, -0.00001407]])
+    array([[ 0.00017619,  0.00004014],
+           [ 0.00010268,  0.00020546],
+           [ 0.00018298, -0.00008335],
+           [ 0.00009501, -0.00001388]])
 
     References
     ----------
diff --git a/sklearn/manifold/tests/test_t_sne.py b/sklearn/manifold/tests/test_t_sne.py
index ea9037776d71ec013f240d779b1f8c1a3159bc4f..52c056a5adadfa4eb3f896bb029a442ed0ae8958 100644
--- a/sklearn/manifold/tests/test_t_sne.py
+++ b/sklearn/manifold/tests/test_t_sne.py
@@ -12,6 +12,7 @@ from sklearn.utils.testing import assert_array_almost_equal
 from sklearn.utils.testing import assert_less
 from sklearn.utils.testing import assert_raises_regexp
 from sklearn.utils.testing import assert_in
+from sklearn.utils.testing import skip_if_32bit
 from sklearn.utils import check_random_state
 from sklearn.manifold.t_sne import _joint_probabilities
 from sklearn.manifold.t_sne import _joint_probabilities_nn
@@ -563,12 +564,13 @@ def test_index_offset():
     assert_equal(_barnes_hut_tsne.test_index_offset(), 1)
 
 
+@skip_if_32bit
 def test_n_iter_without_progress():
-    # Make sure that the parameter n_iter_without_progress is used correctly
+    # Use a dummy negative n_iter_without_progress and check output on stdout
     random_state = check_random_state(0)
     X = random_state.randn(100, 2)
-    tsne = TSNE(n_iter_without_progress=2, verbose=2,
-                random_state=0, method='exact')
+    tsne = TSNE(n_iter_without_progress=-1, verbose=2,
+                random_state=1, method='exact')
 
     old_stdout = sys.stdout
     sys.stdout = StringIO()
@@ -581,7 +583,7 @@ def test_n_iter_without_progress():
 
     # The output needs to contain the value of n_iter_without_progress
     assert_in("did not make any progress during the "
-              "last 2 episodes. Finished.", out)
+              "last -1 episodes. Finished.", out)
 
 
 def test_min_grad_norm():