Feat: Add rope scaling (#343)

* Feat: Add rope scaling * fix: move rope config
2023-08-13 00:50:15 +09:00
parent 289d5c403d
commit b5212068ac
2 changed files with 7 additions and 1 deletions
--- a/src/axolotl/utils/models.py
+++ b/src/axolotl/utils/models.py
@@ -219,7 +219,9 @@ def load_model(
        elif cfg.is_llama_derived_model and not cfg.trust_remote_code:
            from transformers import LlamaForCausalLM

-            config = LlamaConfig.from_pretrained(base_model_config)
+            config = LlamaConfig.from_pretrained(
+                base_model_config, rope_scaling=cfg.rope_scaling
+            )
            model = LlamaForCausalLM.from_pretrained(
                base_model,
                config=config,