tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Dan Saunders
2024-12-27 21:29:37 +00:00
parent 78e0ec0aa5
commit e5fa842ff8
1 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/axolotl/integrations/diff_transformer/modeling_diff_attn.py
View File

@@ -25,6 +25,8 @@ logger = logging.getLogger(__name__)
class LlamaDifferentialConfig(LlamaConfig): class LlamaDifferentialConfig(LlamaConfig):
"""Configuration class for Differential LLaMA model.""" """Configuration class for Differential LLaMA model."""
model_type = "llama-differential"
def __init__( def __init__(
self, self,
split_heads: bool = False, split_heads: bool = False,
@@ -213,6 +215,9 @@ class LlamaDifferentialModel(LlamaModel):
class LlamaDifferentialForCausalLM(LlamaForCausalLM): class LlamaDifferentialForCausalLM(LlamaForCausalLM):
"""LlamaForCausalLM with differential attention.""" """LlamaForCausalLM with differential attention."""
config_class = LlamaDifferentialConfig
base_model_prefix = "llama_differential"
def __init__(self, config): def __init__(self, config):
super().__init__(config) super().__init__(config)
self.model = LlamaDifferentialModel(config) self.model = LlamaDifferentialModel(config)