From b3f680d305528f94c3d0663dc825cc17fcabfea3 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Fri, 5 Jul 2024 06:24:07 -0700 Subject: [PATCH] sanity check ranges in freeze.py (#1686) * sanity check ranges in freeze.py this will catch problems earlier and more clearly. in my case, it appears that deepspeed zero3 sets layer tensor shapes to [0], which doesn't play well with automatically inferred ranges. through a bit of luck, inverting ranges still appears to work correctly. * simplify chained comparison --- src/axolotl/utils/freeze.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/axolotl/utils/freeze.py b/src/axolotl/utils/freeze.py index e3d0fd144..f96bc120d 100644 --- a/src/axolotl/utils/freeze.py +++ b/src/axolotl/utils/freeze.py @@ -120,6 +120,9 @@ def _merge_ranges( processed_ranges = [ (start, end if end is not None else layer_size) for start, end in given_ranges ] + for start, end in processed_ranges: + if start < 0 or end > layer_size > 0 or start >= end: + raise ValueError(f"invalid unfreeze range: start={start}, end={end}") # No need to merge if there's only one or no ranges if len(processed_ranges) <= 1: