diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp
index ce6969a41..977eccf6f 100644
--- a/image_analysis/structure_refinement/Refinement.cpp
+++ b/image_analysis/structure_refinement/Refinement.cpp
@@ -540,9 +540,10 @@ public:
                 for (int c = 0; c < 3; c++)
                     gradient[3 * k + c] = weight_ * gx[i][c] + gr[i][c];
             else {
-                // held at a bound, B does not move further out
-                const bool held = (p[k] <= ctx_.b_floor && gx[i][3] > 0) || (p[k] >= ctx_.b_ceiling && gx[i][3] < 0);
-                gradient[k] = held ? 0.0 : weight_ * gx[i][3] + gr[i][3];
+                // held at a bound, B does not move further out: by the whole target, restraints included
+                const double g = weight_ * gx[i][3] + gr[i][3];
+                const bool held = (p[k] <= ctx_.b_floor && g > 0) || (p[k] >= ctx_.b_ceiling && g < 0);
+                gradient[k] = held ? 0.0 : g;
             }
         }
         return std::isfinite(*cost);