diff --git a/tests/python/unittest/test_optimizer.py b/tests/python/unittest/test_optimizer.py
index d3545ce86638..9ee63d52e50c 100755
--- a/tests/python/unittest/test_optimizer.py
+++ b/tests/python/unittest/test_optimizer.py
@@ -687,9 +687,10 @@ def test_ftrl():
             if (dtype == np.float16 and
                     ('multi_precision' not in kwarg or not kwarg['multi_precision'])):
                 continue
+            rtol, atol = (1e-3, 1e-3) if dtype is np.float16 else (1e-4, 1e-4)
             compare_optimizer(opt1(use_fused_step=False, **kwarg),
                               opt2(use_fused_step=True, **kwarg), shapes, dtype,
-                              rtol=1e-4, atol=1e-4)
+                              rtol=rtol, atol=atol)
 
 
 @with_seed()