From d538cbf716607f9b53a348c8dfa202363575abf0 Mon Sep 17 00:00:00 2001 From: Aaron Fenyes Date: Wed, 10 Jul 2024 23:31:44 -0700 Subject: [PATCH] Correct improvement threshold by using unit step Our formula for the improvement theshold works when the step size is an absolute distance. However, in commit `4d5ea06`, the step size was measured relative to the current gradient instead. This commit scales the base step to unit length, so now the step size really is an absolute distance. --- engine-proto/gram-test/Engine.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine-proto/gram-test/Engine.jl b/engine-proto/gram-test/Engine.jl index 0539326..a160291 100644 --- a/engine-proto/gram-test/Engine.jl +++ b/engine-proto/gram-test/Engine.jl @@ -120,6 +120,7 @@ function realize_gram( # find negative gradient of loss function neg_grad = 4*Q*L*Δ_proj slope = norm(neg_grad) + dir = neg_grad / slope # store current position, loss, and slope L_last = L @@ -135,7 +136,7 @@ function realize_gram( empty!(history.last_line_loss) for backoff_steps in 0:max_backoff_steps history.stepsize[end] = stepsize - L = L_last + stepsize * neg_grad + L = L_last + stepsize * dir Δ_proj = proj_diff(gram, L'*Q*L) loss = dot(Δ_proj, Δ_proj) improvement = loss_last - loss