Feat: Add rope scaling (#343)

* Feat: Add rope scaling * fix: move rope config
2023-08-13 00:50:15 +09:00
parent 289d5c403d
commit b5212068ac
2 changed files with 7 additions and 1 deletions
--- a/README.md
+++ b/README.md
@@ -474,6 +474,10 @@ landmark_attention:
 # xpos RoPE see https://github.com/kaiokendev/cutoff-len-is-context-len/blob/main/util/xpos_rope_llama_monkey_patch.py
 # llama only
 xpos_rope:
+# RoPE Scaling https://github.com/huggingface/transformers/pull/24653
+rope_scaling:
+  type: # linear | dynamic
+  factor: # float

 # resume from a specific checkpoint dir
 resume_from_checkpoint: