add gptneox embeddings, fix phi2 inputs, also fix the casting (#1083)

2024-01-10 22:32:43 -05:00
parent 23495a80af
commit 78c5b1979e
4 changed files with 10 additions and 9 deletions
--- a/tests/test_validation.py
+++ b/tests/test_validation.py
@@ -770,7 +770,7 @@ class ValidationCheckModelConfig(BaseValidation):
                "adapter": "qlora",
                "load_in_4bit": True,
                "tokens": ["<|imstart|>"],
-                "lora_modules_to_save": ["embd", "lm_head.linear"],
+                "lora_modules_to_save": ["embd.wte", "lm_head.linear"],
            }
        )