@@ -1,938 +0,0 @@
<!DOCTYPE html>
< html xmlns = "http://www.w3.org/1999/xhtml" lang = "en" xml:lang = "en" > < head >
< meta charset = "utf-8" >
< meta name = "generator" content = "quarto-1.7.31" >
< meta name = "viewport" content = "width=device-width, initial-scale=1.0, user-scalable=yes" >
< title > monkeypatch.attention.mllama – < / title >
< style >
code { white-space : pre-wrap ; }
span . smallcaps { font-variant : small-caps ; }
div . columns { display : flex ; gap : min ( 4 vw , 1.5 em ) ; }
div . column { flex : auto ; overflow-x : auto ; }
div . hanging-indent { margin-left : 1.5 em ; text-indent : -1.5 em ; }
ul . task-list { list-style : none ; }
ul . task-list li input [ type = "checkbox" ] {
width : 0.8 em ;
margin : 0 0.8 em 0.2 em -1 em ; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
vertical-align : middle ;
}
/* CSS for syntax highlighting */
html { -webkit- text-size-adjust : 100 % ; }
pre > code . sourceCode { white-space : pre ; position : relative ; }
pre > code . sourceCode > span { display : inline-block ; line-height : 1.25 ; }
pre > code . sourceCode > span : empty { height : 1.2 em ; }
. sourceCode { overflow : visible ; }
code . sourceCode > span { color : inherit ; text-decoration : inherit ; }
div . sourceCode { margin : 1 em 0 ; }
pre . sourceCode { margin : 0 ; }
@ media screen {
div . sourceCode { overflow : auto ; }
}
@ media print {
pre > code . sourceCode { white-space : pre-wrap ; }
pre > code . sourceCode > span { text-indent : -5 em ; padding-left : 5 em ; }
}
pre . numberSource code
{ counter-reset : source-line 0 ; }
pre . numberSource code > span
{ position : relative ; left : -4 em ; counter-increment : source-line ; }
pre . numberSource code > span > a : first-child :: before
{ content : counter ( source - line ) ;
position : relative ; left : -1 em ; text-align : right ; vertical-align : baseline ;
border : none ; display : inline-block ;
-webkit- touch-callout : none ; -webkit- user-select : none ;
-khtml- user-select : none ; -moz- user-select : none ;
-ms- user-select : none ; user-select : none ;
padding : 0 4 px ; width : 4 em ;
}
pre . numberSource { margin-left : 3 em ; padding-left : 4 px ; }
div . sourceCode
{ }
@ media screen {
pre > code . sourceCode > span > a : first-child :: before { text-decoration : underline ; }
}
< / style >
< script src = "../../site_libs/quarto-nav/quarto-nav.js" > < / script >
< script src = "../../site_libs/clipboard/clipboard.min.js" > < / script >
< script src = "../../site_libs/quarto-search/autocomplete.umd.js" > < / script >
< script src = "../../site_libs/quarto-search/fuse.min.js" > < / script >
< script src = "../../site_libs/quarto-search/quarto-search.js" > < / script >
< meta name = "quarto:offset" content = "../../" >
< link href = "../../favicon.jpg" rel = "icon" type = "image/jpeg" >
< script src = "../../site_libs/quarto-html/quarto.js" type = "module" > < / script >
< script src = "../../site_libs/quarto-html/tabsets/tabsets.js" type = "module" > < / script >
< script src = "../../site_libs/quarto-html/popper.min.js" > < / script >
< script src = "../../site_libs/quarto-html/tippy.umd.min.js" > < / script >
< script src = "../../site_libs/quarto-html/anchor.min.js" > < / script >
< link href = "../../site_libs/quarto-html/tippy.css" rel = "stylesheet" >
< link href = "../../site_libs/quarto-html/quarto-syntax-highlighting-dark-8ef56b68f8fa1e9d2ba328e99e439f80.css" rel = "stylesheet" id = "quarto-text-highlighting-styles" >
< script src = "../../site_libs/bootstrap/bootstrap.min.js" > < / script >
< link href = "../../site_libs/bootstrap/bootstrap-icons.css" rel = "stylesheet" >
< link href = "../../site_libs/bootstrap/bootstrap-7415244d48e501ce6749b46a5d56d2c8.min.css" rel = "stylesheet" append-hash = "true" id = "quarto-bootstrap" data-mode = "dark" >
< script id = "quarto-search-options" type = "application/json" > {
"location" : "navbar" ,
"copy-button" : false ,
"collapse-after" : 3 ,
"panel-placement" : "end" ,
"type" : "overlay" ,
"limit" : 50 ,
"keyboard-shortcut" : [
"f" ,
"/" ,
"s"
] ,
"show-item-context" : false ,
"language" : {
"search-no-results-text" : "No results" ,
"search-matching-documents-text" : "matching documents" ,
"search-copy-link-title" : "Copy link to search" ,
"search-hide-matches-text" : "Hide additional matches" ,
"search-more-match-text" : "more match in this document" ,
"search-more-matches-text" : "more matches in this document" ,
"search-clear-button-title" : "Clear" ,
"search-text-placeholder" : "" ,
"search-detached-cancel-button-title" : "Cancel" ,
"search-submit-button-title" : "Submit" ,
"search-label" : "Search"
}
} < / script >
< script async = "" src = "https://www.googletagmanager.com/gtag/js?id=G-9KYCVJBNMQ" > < / script >
< script type = "text/javascript" >
window . dataLayer = window . dataLayer || [ ] ;
function gtag ( ) { dataLayer . push ( arguments ) ; }
gtag ( 'js' , new Date ( ) ) ;
gtag ( 'config' , 'G-9KYCVJBNMQ' , { 'anonymize_ip' : true } ) ;
< / script >
< link rel = "stylesheet" href = "../../styles.css" >
< / head >
< body class = "nav-sidebar docked nav-fixed quarto-light" >
< div id = "quarto-search-results" > < / div >
< header id = "quarto-header" class = "headroom fixed-top" >
< nav class = "navbar navbar-expand " data-bs-theme = "dark" >
< div class = "navbar-container container-fluid" >
< div class = "navbar-brand-container mx-auto" >
< a href = "../../index.html" class = "navbar-brand navbar-brand-logo" >
< img src = "../../image/axolotl_logo_digital_white.svg" alt = "" class = "navbar-logo" >
< / a >
< / div >
< div class = "quarto-navbar-tools tools-wide tools-end" >
< a href = "https://twitter.com/axolotl_ai" title = "" class = "quarto-navigation-tool px-1" aria-label = "" > < i class = "bi bi-twitter" > < / i > < / a >
< a href = "https://github.com/axolotl-ai-cloud/axolotl/" title = "" class = "quarto-navigation-tool px-1" aria-label = "" > < i class = "bi bi-github" > < / i > < / a >
< a href = "https://discord.gg/7m9sfhzaf3" title = "" class = "quarto-navigation-tool px-1" aria-label = "" > < i class = "bi bi-discord" > < / i > < / a >
< / div >
< div id = "quarto-search" class = "" title = "Search" > < / div >
< / div > <!-- /container - fluid -->
< / nav >
< nav class = "quarto-secondary-nav" >
< div class = "container-fluid d-flex" >
< button type = "button" class = "quarto-btn-toggle btn" data-bs-toggle = "collapse" role = "button" data-bs-target = ".quarto-sidebar-collapse-item" aria-controls = "quarto-sidebar" aria-expanded = "false" aria-label = "Toggle sidebar navigation" onclick = "if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }" >
< i class = "bi bi-layout-text-sidebar-reverse" > < / i >
< / button >
< nav class = "quarto-page-breadcrumbs" aria-label = "breadcrumb" > < ol class = "breadcrumb" > < / ol > < / nav >
< a class = "flex-grow-1" role = "navigation" data-bs-toggle = "collapse" data-bs-target = ".quarto-sidebar-collapse-item" aria-controls = "quarto-sidebar" aria-expanded = "false" aria-label = "Toggle sidebar navigation" onclick = "if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }" >
< / a >
< / div >
< / nav >
< / header >
<!-- content -->
< div id = "quarto-content" class = "quarto-container page-columns page-rows-contents page-layout-article page-navbar" >
<!-- sidebar -->
< nav id = "quarto-sidebar" class = "sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation docked overflow-auto" >
< div class = "sidebar-menu-container" >
< ul class = "list-unstyled mt-1" >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../index.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Home< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item sidebar-item-section" >
< div class = "sidebar-item-container" >
< a class = "sidebar-item-text sidebar-link text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-1" role = "navigation" aria-expanded = "true" >
< span class = "menu-text" > Getting Started< / span > < / a >
< a class = "sidebar-item-toggle text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-1" role = "navigation" aria-expanded = "true" aria-label = "Toggle section" >
< i class = "bi bi-chevron-right ms-2" > < / i >
< / a >
< / div >
< ul id = "quarto-sidebar-section-1" class = "collapse list-unstyled sidebar-section depth1 show" >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/getting-started.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Quickstart< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/installation.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Installation< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/inference.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Inference and Merging< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/cli.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Command Line Interface (CLI)< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/config.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Config Reference< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/api" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > API Reference< / span > < / a >
< / div >
< / li >
< / ul >
< / li >
< li class = "sidebar-item sidebar-item-section" >
< div class = "sidebar-item-container" >
< a href = "../../docs/dataset-formats/index.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Dataset Formats< / span > < / a >
< a class = "sidebar-item-toggle text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-2" role = "navigation" aria-expanded = "true" aria-label = "Toggle section" >
< i class = "bi bi-chevron-right ms-2" > < / i >
< / a >
< / div >
< ul id = "quarto-sidebar-section-2" class = "collapse list-unstyled sidebar-section depth1 show" >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/dataset-formats/pretraining.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Pre-training< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/dataset-formats/inst_tune.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Instruction Tuning< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/dataset-formats/conversation.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Conversation< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/dataset-formats/stepwise_supervised.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Stepwise Supervised Format< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/dataset-formats/template_free.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Template-Free< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/dataset-formats/tokenized.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Custom Pre-Tokenized Dataset< / span > < / a >
< / div >
< / li >
< / ul >
< / li >
< li class = "sidebar-item sidebar-item-section" >
< div class = "sidebar-item-container" >
< a class = "sidebar-item-text sidebar-link text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-3" role = "navigation" aria-expanded = "true" >
< span class = "menu-text" > Deployments< / span > < / a >
< a class = "sidebar-item-toggle text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-3" role = "navigation" aria-expanded = "true" aria-label = "Toggle section" >
< i class = "bi bi-chevron-right ms-2" > < / i >
< / a >
< / div >
< ul id = "quarto-sidebar-section-3" class = "collapse list-unstyled sidebar-section depth1 show" >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/docker.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Docker< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/multi-gpu.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Multi-GPU< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/multi-node.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Multi Node< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/ray-integration.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Ray Train< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/amd_hpc.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > AMD GPUs on HPC Systems< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/mac.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Mac M-series< / span > < / a >
< / div >
< / li >
< / ul >
< / li >
< li class = "sidebar-item sidebar-item-section" >
< div class = "sidebar-item-container" >
< a class = "sidebar-item-text sidebar-link text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-4" role = "navigation" aria-expanded = "true" >
< span class = "menu-text" > How To Guides< / span > < / a >
< a class = "sidebar-item-toggle text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-4" role = "navigation" aria-expanded = "true" aria-label = "Toggle section" >
< i class = "bi bi-chevron-right ms-2" > < / i >
< / a >
< / div >
< ul id = "quarto-sidebar-section-4" class = "collapse list-unstyled sidebar-section depth1 show" >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/multimodal.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > MultiModal / Vision Language Models (BETA)< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/rlhf.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > RLHF (Beta)< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/reward_modelling.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Reward Modelling< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/lr_groups.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Learning Rate Groups< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/lora_optims.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > LoRA Optimizations< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/dataset_loading.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Dataset Loading< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/qat.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Quantization Aware Training (QAT)< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/quantize.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Quantization with torchao< / span > < / a >
< / div >
< / li >
< / ul >
< / li >
< li class = "sidebar-item sidebar-item-section" >
< div class = "sidebar-item-container" >
< a class = "sidebar-item-text sidebar-link text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-5" role = "navigation" aria-expanded = "true" >
< span class = "menu-text" > Core Concepts< / span > < / a >
< a class = "sidebar-item-toggle text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-5" role = "navigation" aria-expanded = "true" aria-label = "Toggle section" >
< i class = "bi bi-chevron-right ms-2" > < / i >
< / a >
< / div >
< ul id = "quarto-sidebar-section-5" class = "collapse list-unstyled sidebar-section depth1 show" >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/batch_vs_grad.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Batch size vs Gradient accumulation< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/dataset_preprocessing.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Dataset Preprocessing< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/multipack.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Multipack (Sample Packing)< / span > < / a >
< / div >
< / li >
< / ul >
< / li >
< li class = "sidebar-item sidebar-item-section" >
< div class = "sidebar-item-container" >
< a class = "sidebar-item-text sidebar-link text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-6" role = "navigation" aria-expanded = "true" >
< span class = "menu-text" > Advanced Features< / span > < / a >
< a class = "sidebar-item-toggle text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-6" role = "navigation" aria-expanded = "true" aria-label = "Toggle section" >
< i class = "bi bi-chevron-right ms-2" > < / i >
< / a >
< / div >
< ul id = "quarto-sidebar-section-6" class = "collapse list-unstyled sidebar-section depth1 show" >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/fsdp_qlora.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > FDSP + QLoRA< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/unsloth.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Unsloth< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/torchao.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > PyTorch ao< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/custom_integrations.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Custom Integrations< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/sequence_parallelism.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Sequence Parallelism< / span > < / a >
< / div >
< / li >
< / ul >
< / li >
< li class = "sidebar-item sidebar-item-section" >
< div class = "sidebar-item-container" >
< a class = "sidebar-item-text sidebar-link text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-7" role = "navigation" aria-expanded = "true" >
< span class = "menu-text" > Troubleshooting< / span > < / a >
< a class = "sidebar-item-toggle text-start" data-bs-toggle = "collapse" data-bs-target = "#quarto-sidebar-section-7" role = "navigation" aria-expanded = "true" aria-label = "Toggle section" >
< i class = "bi bi-chevron-right ms-2" > < / i >
< / a >
< / div >
< ul id = "quarto-sidebar-section-7" class = "collapse list-unstyled sidebar-section depth1 show" >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/faq.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > FAQ< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/debugging.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > Debugging< / span > < / a >
< / div >
< / li >
< li class = "sidebar-item" >
< div class = "sidebar-item-container" >
< a href = "../../docs/nccl.html" class = "sidebar-item-text sidebar-link" >
< span class = "menu-text" > NCCL< / span > < / a >
< / div >
< / li >
< / ul >
< / li >
< / ul >
< / div >
< / nav >
< div id = "quarto-sidebar-glass" class = "quarto-sidebar-collapse-item" data-bs-toggle = "collapse" data-bs-target = ".quarto-sidebar-collapse-item" > < / div >
<!-- margin - sidebar -->
< div id = "quarto-margin-sidebar" class = "sidebar margin-sidebar" >
< nav id = "TOC" role = "doc-toc" class = "toc-active" >
< h2 id = "toc-title" > On this page< / h2 >
< ul >
< li > < a href = "#axolotl.monkeypatch.attention.mllama" id = "toc-axolotl.monkeypatch.attention.mllama" class = "nav-link active" data-scroll-target = "#axolotl.monkeypatch.attention.mllama" > monkeypatch.attention.mllama< / a >
< ul class = "collapse" >
< li > < a href = "#classes" id = "toc-classes" class = "nav-link" data-scroll-target = "#classes" > Classes< / a >
< ul class = "collapse" >
< li > < a href = "#axolotl.monkeypatch.attention.mllama.MllamaTextCrossFlashAttention2" id = "toc-axolotl.monkeypatch.attention.mllama.MllamaTextCrossFlashAttention2" class = "nav-link" data-scroll-target = "#axolotl.monkeypatch.attention.mllama.MllamaTextCrossFlashAttention2" > MllamaTextCrossFlashAttention2< / a > < / li >
< li > < a href = "#axolotl.monkeypatch.attention.mllama.MllamaTextSelfFlashAttention2" id = "toc-axolotl.monkeypatch.attention.mllama.MllamaTextSelfFlashAttention2" class = "nav-link" data-scroll-target = "#axolotl.monkeypatch.attention.mllama.MllamaTextSelfFlashAttention2" > MllamaTextSelfFlashAttention2< / a > < / li >
< / ul > < / li >
< / ul > < / li >
< / ul >
< / nav >
< / div >
<!-- main -->
< main class = "content" id = "quarto-document-content" > < header id = "title-block-header" class = "quarto-title-block" > < / header >
< section id = "axolotl.monkeypatch.attention.mllama" class = "level1" >
< h1 > monkeypatch.attention.mllama< / h1 >
< p > < code > monkeypatch.attention.mllama< / code > < / p >
< p > Monkeypatch for Vision Llama for FA2 support< / p >
< section id = "classes" class = "level2" >
< h2 class = "anchored" data-anchor-id = "classes" > Classes< / h2 >
< table class = "caption-top table" >
< thead >
< tr class = "header" >
< th > Name< / th >
< th > Description< / th >
< / tr >
< / thead >
< tbody >
< tr class = "odd" >
< td > < a href = "#axolotl.monkeypatch.attention.mllama.MllamaTextCrossFlashAttention2" > MllamaTextCrossFlashAttention2< / a > < / td >
< td > Mllama flash cross-attention module. This module inherits from < code > MllamaTextCrossAttention< / code > and< / td >
< / tr >
< tr class = "even" >
< td > < a href = "#axolotl.monkeypatch.attention.mllama.MllamaTextSelfFlashAttention2" > MllamaTextSelfFlashAttention2< / a > < / td >
< td > Mllama flash self-attention module. This module inherits from < code > MllamaTextSelfAttention< / code > and< / td >
< / tr >
< / tbody >
< / table >
< section id = "axolotl.monkeypatch.attention.mllama.MllamaTextCrossFlashAttention2" class = "level3" >
< h3 class = "anchored" data-anchor-id = "axolotl.monkeypatch.attention.mllama.MllamaTextCrossFlashAttention2" > MllamaTextCrossFlashAttention2< / h3 >
< div class = "sourceCode" id = "cb1" > < pre class = "sourceCode python code-with-copy" > < code class = "sourceCode python" > < span id = "cb1-1" > < a href = "#cb1-1" aria-hidden = "true" tabindex = "-1" > < / a > monkeypatch.attention.mllama.MllamaTextCrossFlashAttention2(< span class = "op" > *< / span > args, < span class = "op" > **< / span > kwargs)< / span > < / code > < button title = "Copy to Clipboard" class = "code-copy-button" > < i class = "bi" > < / i > < / button > < / pre > < / div >
< p > Mllama flash cross-attention module. This module inherits from < code > MllamaTextCrossAttention< / code > and
implements the forward pass using Flash Attention for improved performance.< / p >
< / section >
< section id = "axolotl.monkeypatch.attention.mllama.MllamaTextSelfFlashAttention2" class = "level3" >
< h3 class = "anchored" data-anchor-id = "axolotl.monkeypatch.attention.mllama.MllamaTextSelfFlashAttention2" > MllamaTextSelfFlashAttention2< / h3 >
< div class = "sourceCode" id = "cb2" > < pre class = "sourceCode python code-with-copy" > < code class = "sourceCode python" > < span id = "cb2-1" > < a href = "#cb2-1" aria-hidden = "true" tabindex = "-1" > < / a > monkeypatch.attention.mllama.MllamaTextSelfFlashAttention2(< / span >
< span id = "cb2-2" > < a href = "#cb2-2" aria-hidden = "true" tabindex = "-1" > < / a > config,< / span >
< span id = "cb2-3" > < a href = "#cb2-3" aria-hidden = "true" tabindex = "-1" > < / a > layer_idx,< / span >
< span id = "cb2-4" > < a href = "#cb2-4" aria-hidden = "true" tabindex = "-1" > < / a > < span class = "op" > *< / span > args,< / span >
< span id = "cb2-5" > < a href = "#cb2-5" aria-hidden = "true" tabindex = "-1" > < / a > < span class = "op" > **< / span > kwargs,< / span >
< span id = "cb2-6" > < a href = "#cb2-6" aria-hidden = "true" tabindex = "-1" > < / a > )< / span > < / code > < button title = "Copy to Clipboard" class = "code-copy-button" > < i class = "bi" > < / i > < / button > < / pre > < / div >
< p > Mllama flash self-attention module. This module inherits from < code > MllamaTextSelfAttention< / code > and
implements the forward pass using Flash Attention for improved performance.< / p >
< / section >
< / section >
< / section >
< / main > <!-- /main -->
< script id = "quarto-html-after-body" type = "application/javascript" >
window . document . addEventListener ( "DOMContentLoaded" , function ( event ) {
const icon = "" ;
const anchorJS = new window . AnchorJS ( ) ;
anchorJS . options = {
placement : 'right' ,
icon : icon
} ;
anchorJS . add ( '.anchored' ) ;
const isCodeAnnotation = ( el ) => {
for ( const clz of el . classList ) {
if ( clz . startsWith ( 'code-annotation-' ) ) {
return true ;
}
}
return false ;
}
const onCopySuccess = function ( e ) {
// button target
const button = e . trigger ;
// don't keep focus
button . blur ( ) ;
// flash "checked"
button . classList . add ( 'code-copy-button-checked' ) ;
var currentTitle = button . getAttribute ( "title" ) ;
button . setAttribute ( "title" , "Copied!" ) ;
let tooltip ;
if ( window . bootstrap ) {
button . setAttribute ( "data-bs-toggle" , "tooltip" ) ;
button . setAttribute ( "data-bs-placement" , "left" ) ;
button . setAttribute ( "data-bs-title" , "Copied!" ) ;
tooltip = new bootstrap . Tooltip ( button ,
{ trigger : "manual" ,
customClass : "code-copy-button-tooltip" ,
offset : [ 0 , - 8 ] } ) ;
tooltip . show ( ) ;
}
setTimeout ( function ( ) {
if ( tooltip ) {
tooltip . hide ( ) ;
button . removeAttribute ( "data-bs-title" ) ;
button . removeAttribute ( "data-bs-toggle" ) ;
button . removeAttribute ( "data-bs-placement" ) ;
}
button . setAttribute ( "title" , currentTitle ) ;
button . classList . remove ( 'code-copy-button-checked' ) ;
} , 1000 ) ;
// clear code selection
e . clearSelection ( ) ;
}
const getTextToCopy = function ( trigger ) {
const codeEl = trigger . previousElementSibling . cloneNode ( true ) ;
for ( const childEl of codeEl . children ) {
if ( isCodeAnnotation ( childEl ) ) {
childEl . remove ( ) ;
}
}
return codeEl . innerText ;
}
const clipboard = new window . ClipboardJS ( '.code-copy-button:not([data-in-quarto-modal])' , {
text : getTextToCopy
} ) ;
clipboard . on ( 'success' , onCopySuccess ) ;
if ( window . document . getElementById ( 'quarto-embedded-source-code-modal' ) ) {
const clipboardModal = new window . ClipboardJS ( '.code-copy-button[data-in-quarto-modal]' , {
text : getTextToCopy ,
container : window . document . getElementById ( 'quarto-embedded-source-code-modal' )
} ) ;
clipboardModal . on ( 'success' , onCopySuccess ) ;
}
var localhostRegex = new RegExp ( /^(?:http|https):\/\/localhost\:?[0-9]*\// ) ;
var mailtoRegex = new RegExp ( /^mailto:/ ) ;
var filterRegex = new RegExp ( "https:\/\/docs\.axolotl\.ai" ) ;
var isInternal = ( href ) => {
return filterRegex . test ( href ) || localhostRegex . test ( href ) || mailtoRegex . test ( href ) ;
}
// Inspect non-navigation links and adorn them if external
var links = window . document . querySelectorAll ( 'a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)' ) ;
for ( var i = 0 ; i < links . length ; i ++ ) {
const link = links [ i ] ;
if ( ! isInternal ( link . href ) ) {
// undo the damage that might have been done by quarto-nav.js in the case of
// links that we want to consider external
if ( link . dataset . originalHref !== undefined ) {
link . href = link . dataset . originalHref ;
}
}
}
function tippyHover ( el , contentFn , onTriggerFn , onUntriggerFn ) {
const config = {
allowHTML : true ,
maxWidth : 500 ,
delay : 100 ,
arrow : false ,
appendTo : function ( el ) {
return el . parentElement ;
} ,
interactive : true ,
interactiveBorder : 10 ,
theme : 'quarto' ,
placement : 'bottom-start' ,
} ;
if ( contentFn ) {
config . content = contentFn ;
}
if ( onTriggerFn ) {
config . onTrigger = onTriggerFn ;
}
if ( onUntriggerFn ) {
config . onUntrigger = onUntriggerFn ;
}
window . tippy ( el , config ) ;
}
const noterefs = window . document . querySelectorAll ( 'a[role="doc-noteref"]' ) ;
for ( var i = 0 ; i < noterefs . length ; i ++ ) {
const ref = noterefs [ i ] ;
tippyHover ( ref , function ( ) {
// use id or data attribute instead here
let href = ref . getAttribute ( 'data-footnote-href' ) || ref . getAttribute ( 'href' ) ;
try { href = new URL ( href ) . hash ; } catch { }
const id = href . replace ( /^#\/?/ , "" ) ;
const note = window . document . getElementById ( id ) ;
if ( note ) {
return note . innerHTML ;
} else {
return "" ;
}
} ) ;
}
const xrefs = window . document . querySelectorAll ( 'a.quarto-xref' ) ;
const processXRef = ( id , note ) => {
// Strip column container classes
const stripColumnClz = ( el ) => {
el . classList . remove ( "page-full" , "page-columns" ) ;
if ( el . children ) {
for ( const child of el . children ) {
stripColumnClz ( child ) ;
}
}
}
stripColumnClz ( note )
if ( id === null || id . startsWith ( 'sec-' ) ) {
// Special case sections, only their first couple elements
const container = document . createElement ( "div" ) ;
if ( note . children && note . children . length > 2 ) {
container . appendChild ( note . children [ 0 ] . cloneNode ( true ) ) ;
for ( let i = 1 ; i < note . children . length ; i ++ ) {
const child = note . children [ i ] ;
if ( child . tagName === "P" && child . innerText === "" ) {
continue ;
} else {
container . appendChild ( child . cloneNode ( true ) ) ;
break ;
}
}
if ( window . Quarto ? . typesetMath ) {
window . Quarto . typesetMath ( container ) ;
}
return container . innerHTML
} else {
if ( window . Quarto ? . typesetMath ) {
window . Quarto . typesetMath ( note ) ;
}
return note . innerHTML ;
}
} else {
// Remove any anchor links if they are present
const anchorLink = note . querySelector ( 'a.anchorjs-link' ) ;
if ( anchorLink ) {
anchorLink . remove ( ) ;
}
if ( window . Quarto ? . typesetMath ) {
window . Quarto . typesetMath ( note ) ;
}
if ( note . classList . contains ( "callout" ) ) {
return note . outerHTML ;
} else {
return note . innerHTML ;
}
}
}
for ( var i = 0 ; i < xrefs . length ; i ++ ) {
const xref = xrefs [ i ] ;
tippyHover ( xref , undefined , function ( instance ) {
instance . disable ( ) ;
let url = xref . getAttribute ( 'href' ) ;
let hash = undefined ;
if ( url . startsWith ( '#' ) ) {
hash = url ;
} else {
try { hash = new URL ( url ) . hash ; } catch { }
}
if ( hash ) {
const id = hash . replace ( /^#\/?/ , "" ) ;
const note = window . document . getElementById ( id ) ;
if ( note !== null ) {
try {
const html = processXRef ( id , note . cloneNode ( true ) ) ;
instance . setContent ( html ) ;
} finally {
instance . enable ( ) ;
instance . show ( ) ;
}
} else {
// See if we can fetch this
fetch ( url . split ( '#' ) [ 0 ] )
. then ( res => res . text ( ) )
. then ( html => {
const parser = new DOMParser ( ) ;
const htmlDoc = parser . parseFromString ( html , "text/html" ) ;
const note = htmlDoc . getElementById ( id ) ;
if ( note !== null ) {
const html = processXRef ( id , note ) ;
instance . setContent ( html ) ;
}
} ) . finally ( ( ) => {
instance . enable ( ) ;
instance . show ( ) ;
} ) ;
}
} else {
// See if we can fetch a full url (with no hash to target)
// This is a special case and we should probably do some content thinning / targeting
fetch ( url )
. then ( res => res . text ( ) )
. then ( html => {
const parser = new DOMParser ( ) ;
const htmlDoc = parser . parseFromString ( html , "text/html" ) ;
const note = htmlDoc . querySelector ( 'main.content' ) ;
if ( note !== null ) {
// This should only happen for chapter cross references
// (since there is no id in the URL)
// remove the first header
if ( note . children . length > 0 && note . children [ 0 ] . tagName === "HEADER" ) {
note . children [ 0 ] . remove ( ) ;
}
const html = processXRef ( null , note ) ;
instance . setContent ( html ) ;
}
} ) . finally ( ( ) => {
instance . enable ( ) ;
instance . show ( ) ;
} ) ;
}
} , function ( instance ) {
} ) ;
}
let selectedAnnoteEl ;
const selectorForAnnotation = ( cell , annotation ) => {
let cellAttr = 'data-code-cell="' + cell + '"' ;
let lineAttr = 'data-code-annotation="' + annotation + '"' ;
const selector = 'span[' + cellAttr + '][' + lineAttr + ']' ;
return selector ;
}
const selectCodeLines = ( annoteEl ) => {
const doc = window . document ;
const targetCell = annoteEl . getAttribute ( "data-target-cell" ) ;
const targetAnnotation = annoteEl . getAttribute ( "data-target-annotation" ) ;
const annoteSpan = window . document . querySelector ( selectorForAnnotation ( targetCell , targetAnnotation ) ) ;
const lines = annoteSpan . getAttribute ( "data-code-lines" ) . split ( "," ) ;
const lineIds = lines . map ( ( line ) => {
return targetCell + "-" + line ;
} )
let top = null ;
let height = null ;
let parent = null ;
if ( lineIds . length > 0 ) {
//compute the position of the single el (top and bottom and make a div)
const el = window . document . getElementById ( lineIds [ 0 ] ) ;
top = el . offsetTop ;
height = el . offsetHeight ;
parent = el . parentElement . parentElement ;
if ( lineIds . length > 1 ) {
const lastEl = window . document . getElementById ( lineIds [ lineIds . length - 1 ] ) ;
const bottom = lastEl . offsetTop + lastEl . offsetHeight ;
height = bottom - top ;
}
if ( top !== null && height !== null && parent !== null ) {
// cook up a div (if necessary) and position it
let div = window . document . getElementById ( "code-annotation-line-highlight" ) ;
if ( div === null ) {
div = window . document . createElement ( "div" ) ;
div . setAttribute ( "id" , "code-annotation-line-highlight" ) ;
div . style . position = 'absolute' ;
parent . appendChild ( div ) ;
}
div . style . top = top - 2 + "px" ;
div . style . height = height + 4 + "px" ;
div . style . left = 0 ;
let gutterDiv = window . document . getElementById ( "code-annotation-line-highlight-gutter" ) ;
if ( gutterDiv === null ) {
gutterDiv = window . document . createElement ( "div" ) ;
gutterDiv . setAttribute ( "id" , "code-annotation-line-highlight-gutter" ) ;
gutterDiv . style . position = 'absolute' ;
const codeCell = window . document . getElementById ( targetCell ) ;
const gutter = codeCell . querySelector ( '.code-annotation-gutter' ) ;
gutter . appendChild ( gutterDiv ) ;
}
gutterDiv . style . top = top - 2 + "px" ;
gutterDiv . style . height = height + 4 + "px" ;
}
selectedAnnoteEl = annoteEl ;
}
} ;
const unselectCodeLines = ( ) => {
const elementsIds = [ "code-annotation-line-highlight" , "code-annotation-line-highlight-gutter" ] ;
elementsIds . forEach ( ( elId ) => {
const div = window . document . getElementById ( elId ) ;
if ( div ) {
div . remove ( ) ;
}
} ) ;
selectedAnnoteEl = undefined ;
} ;
// Handle positioning of the toggle
window . addEventListener (
"resize" ,
throttle ( ( ) => {
elRect = undefined ;
if ( selectedAnnoteEl ) {
selectCodeLines ( selectedAnnoteEl ) ;
}
} , 10 )
) ;
function throttle ( fn , ms ) {
let throttle = false ;
let timer ;
return ( ... args ) => {
if ( ! throttle ) { // first call gets through
fn . apply ( this , args ) ;
throttle = true ;
} else { // all the others get throttled
if ( timer ) clearTimeout ( timer ) ; // cancel #2
timer = setTimeout ( ( ) => {
fn . apply ( this , args ) ;
timer = throttle = false ;
} , ms ) ;
}
} ;
}
// Attach click handler to the DT
const annoteDls = window . document . querySelectorAll ( 'dt[data-target-cell]' ) ;
for ( const annoteDlNode of annoteDls ) {
annoteDlNode . addEventListener ( 'click' , ( event ) => {
const clickedEl = event . target ;
if ( clickedEl !== selectedAnnoteEl ) {
unselectCodeLines ( ) ;
const activeEl = window . document . querySelector ( 'dt[data-target-cell].code-annotation-active' ) ;
if ( activeEl ) {
activeEl . classList . remove ( 'code-annotation-active' ) ;
}
selectCodeLines ( clickedEl ) ;
clickedEl . classList . add ( 'code-annotation-active' ) ;
} else {
// Unselect the line
unselectCodeLines ( ) ;
clickedEl . classList . remove ( 'code-annotation-active' ) ;
}
} ) ;
}
const findCites = ( el ) => {
const parentEl = el . parentElement ;
if ( parentEl ) {
const cites = parentEl . dataset . cites ;
if ( cites ) {
return {
el ,
cites : cites . split ( ' ' )
} ;
} else {
return findCites ( el . parentElement )
}
} else {
return undefined ;
}
} ;
var bibliorefs = window . document . querySelectorAll ( 'a[role="doc-biblioref"]' ) ;
for ( var i = 0 ; i < bibliorefs . length ; i ++ ) {
const ref = bibliorefs [ i ] ;
const citeInfo = findCites ( ref ) ;
if ( citeInfo ) {
tippyHover ( citeInfo . el , function ( ) {
var popup = window . document . createElement ( 'div' ) ;
citeInfo . cites . forEach ( function ( cite ) {
var citeDiv = window . document . createElement ( 'div' ) ;
citeDiv . classList . add ( 'hanging-indent' ) ;
citeDiv . classList . add ( 'csl-entry' ) ;
var biblioDiv = window . document . getElementById ( 'ref-' + cite ) ;
if ( biblioDiv ) {
citeDiv . innerHTML = biblioDiv . innerHTML ;
}
popup . appendChild ( citeDiv ) ;
} ) ;
return popup . innerHTML ;
} ) ;
}
}
} ) ;
< / script >
< / div > <!-- /content -->
< / body > < / html >
Quarto GHA Workflow Runner