diff --git a/.nojekyll b/.nojekyll
index 24d6b6d83..125d97a6d 100644
--- a/.nojekyll
+++ b/.nojekyll
@@ -1 +1 @@
-beb883be
\ No newline at end of file
+a883ffee
\ No newline at end of file
diff --git a/FAQS.html b/FAQS.html
index 04584d57b..24692363d 100644
--- a/FAQS.html
+++ b/FAQS.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
diff --git a/TODO.html b/TODO.html
index 13bd0e233..a21dd09d5 100644
--- a/TODO.html
+++ b/TODO.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
diff --git a/docs/batch_vs_grad.html b/docs/batch_vs_grad.html
index 4c5ad49da..e69c2709b 100644
--- a/docs/batch_vs_grad.html
+++ b/docs/batch_vs_grad.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
diff --git a/docs/config.html b/docs/config.html
index af4fd2cfa..8e76dd3e7 100644
--- a/docs/config.html
+++ b/docs/config.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -34,7 +34,7 @@ div.sourceCode { overflow: auto; }
 }
 @media print {
 pre > code.sourceCode { white-space: pre-wrap; }
-pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
+pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
 }
 pre.numberSource code
   { counter-reset: source-line 0; }
diff --git a/docs/dataset-formats/conversation.html b/docs/dataset-formats/conversation.html
index 6bef9fe6f..097101ece 100644
--- a/docs/dataset-formats/conversation.html
+++ b/docs/dataset-formats/conversation.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -34,7 +34,7 @@ div.sourceCode { overflow: auto; }
 }
 @media print {
 pre > code.sourceCode { white-space: pre-wrap; }
-pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
+pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
 }
 pre.numberSource code
   { counter-reset: source-line 0; }
diff --git a/docs/dataset-formats/index.html b/docs/dataset-formats/index.html
index aba0eca8f..cb1dd012e 100644
--- a/docs/dataset-formats/index.html
+++ b/docs/dataset-formats/index.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -99,7 +99,7 @@ ul.task-list li input[type="checkbox"] {
   })
   </script>
 
-  <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+  <script src="https://cdnjs.cloudflare.com/polyfill/v3/polyfill.min.js?features=es6"></script>
   <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml-full.js" type="text/javascript"></script>
 
 <script type="text/javascript">
@@ -351,7 +351,7 @@ Description
 </tr>
 </thead>
 <tbody class="list">
-<tr data-index="0" data-listing-file-modified-sort="1718892328768" data-listing-reading-time-sort="1" data-listing-word-count-sort="47" data-listing-title-sort="Pre-training" data-listing-filename-sort="pretraining.qmd">
+<tr data-index="0" data-listing-file-modified-sort="1719639545828" data-listing-reading-time-sort="1" data-listing-word-count-sort="47" data-listing-title-sort="Pre-training" data-listing-filename-sort="pretraining.qmd">
 <td>
 <a href="../../docs/dataset-formats/pretraining.html" class="title listing-title">Pre-training</a>
 </td>
@@ -359,7 +359,7 @@ Description
 <span class="listing-description">Data format for a pre-training completion task.</span>
 </td>
 </tr>
-<tr data-index="1" data-listing-file-modified-sort="1718892328768" data-listing-reading-time-sort="2" data-listing-word-count-sort="308" data-listing-title-sort="Instruction Tuning" data-listing-filename-sort="inst_tune.qmd">
+<tr data-index="1" data-listing-file-modified-sort="1719639545828" data-listing-reading-time-sort="2" data-listing-word-count-sort="308" data-listing-title-sort="Instruction Tuning" data-listing-filename-sort="inst_tune.qmd">
 <td>
 <a href="../../docs/dataset-formats/inst_tune.html" class="title listing-title">Instruction Tuning</a>
 </td>
@@ -367,7 +367,7 @@ Description
 <span class="listing-description">Instruction tuning formats for supervised fine-tuning.</span>
 </td>
 </tr>
-<tr data-index="2" data-listing-file-modified-sort="1718892328768" data-listing-reading-time-sort="2" data-listing-word-count-sort="235" data-listing-title-sort="Conversation" data-listing-filename-sort="conversation.qmd">
+<tr data-index="2" data-listing-file-modified-sort="1719639545824" data-listing-reading-time-sort="2" data-listing-word-count-sort="235" data-listing-title-sort="Conversation" data-listing-filename-sort="conversation.qmd">
 <td>
 <a href="../../docs/dataset-formats/conversation.html" class="title listing-title">Conversation</a>
 </td>
@@ -375,7 +375,7 @@ Description
 <span class="listing-description">Conversation format for supervised fine-tuning.</span>
 </td>
 </tr>
-<tr data-index="3" data-listing-file-modified-sort="1718892328768" data-listing-reading-time-sort="1" data-listing-word-count-sort="3" data-listing-title-sort="Template-Free" data-listing-filename-sort="template_free.qmd">
+<tr data-index="3" data-listing-file-modified-sort="1719639545828" data-listing-reading-time-sort="1" data-listing-word-count-sort="3" data-listing-title-sort="Template-Free" data-listing-filename-sort="template_free.qmd">
 <td>
 <a href="../../docs/dataset-formats/template_free.html" class="title listing-title">Template-Free</a>
 </td>
@@ -383,7 +383,7 @@ Description
 <span class="listing-description">Construct prompts without a template.</span>
 </td>
 </tr>
-<tr data-index="4" data-listing-file-modified-sort="1718892328768" data-listing-reading-time-sort="1" data-listing-word-count-sort="22" data-listing-title-sort="Custom Pre-Tokenized Dataset" data-listing-filename-sort="tokenized.qmd">
+<tr data-index="4" data-listing-file-modified-sort="1719639545828" data-listing-reading-time-sort="1" data-listing-word-count-sort="90" data-listing-title-sort="Custom Pre-Tokenized Dataset" data-listing-filename-sort="tokenized.qmd">
 <td>
 <a href="../../docs/dataset-formats/tokenized.html" class="title listing-title">Custom Pre-Tokenized Dataset</a>
 </td>
diff --git a/docs/dataset-formats/inst_tune.html b/docs/dataset-formats/inst_tune.html
index f7a6a41e3..2bc59c9fa 100644
--- a/docs/dataset-formats/inst_tune.html
+++ b/docs/dataset-formats/inst_tune.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -34,7 +34,7 @@ div.sourceCode { overflow: auto; }
 }
 @media print {
 pre > code.sourceCode { white-space: pre-wrap; }
-pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
+pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
 }
 pre.numberSource code
   { counter-reset: source-line 0; }
diff --git a/docs/dataset-formats/pretraining.html b/docs/dataset-formats/pretraining.html
index b5f4dc3f0..2c8c9b1cf 100644
--- a/docs/dataset-formats/pretraining.html
+++ b/docs/dataset-formats/pretraining.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -34,7 +34,7 @@ div.sourceCode { overflow: auto; }
 }
 @media print {
 pre > code.sourceCode { white-space: pre-wrap; }
-pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
+pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
 }
 pre.numberSource code
   { counter-reset: source-line 0; }
diff --git a/docs/dataset-formats/template_free.html b/docs/dataset-formats/template_free.html
index 0f9b74bc0..d089b8f85 100644
--- a/docs/dataset-formats/template_free.html
+++ b/docs/dataset-formats/template_free.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
diff --git a/docs/dataset-formats/tokenized.html b/docs/dataset-formats/tokenized.html
index 2cc470746..96c482a32 100644
--- a/docs/dataset-formats/tokenized.html
+++ b/docs/dataset-formats/tokenized.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -34,7 +34,7 @@ div.sourceCode { overflow: auto; }
 }
 @media print {
 pre > code.sourceCode { white-space: pre-wrap; }
-pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
+pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
 }
 pre.numberSource code
   { counter-reset: source-line 0; }
@@ -307,15 +307,26 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
 
 
 <ul>
-<li>Do not pass a <code>type:</code> in your axolotl config.</li>
+<li>Pass an empty <code>type:</code> in your axolotl config.</li>
 <li>Columns in Dataset must be exactly <code>input_ids</code>, <code>attention_mask</code>, <code>labels</code></li>
+<li>To indicate that a token should be ignored during training, set its corresponding label to <code>-100</code>.</li>
+<li>Do not add BOS/EOS. Axolotl will add them for you based on the default tokenizer for the model you’re using.</li>
+<li>For pretraining, do not truncate/pad documents to the context window length.</li>
+<li>For instruction training, documents must be truncated/padded as desired.</li>
 </ul>
+<p>Sample config:</p>
 <div class="code-with-filename">
 <div class="code-with-filename-file">
 <pre><strong>config.yml</strong></pre>
 </div>
-<div class="sourceCode" id="cb1" data-filename="config.yml"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="kw">-</span><span class="at"> </span><span class="fu">path</span><span class="kw">:</span><span class="at"> ...</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
+<div class="sourceCode" id="cb1" data-filename="config.yml"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="fu">datasets</span><span class="kw">:</span></span>
+<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">path</span><span class="kw">:</span><span class="at"> /path/to/your/file.jsonl</span></span>
+<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">ds_type</span><span class="kw">:</span><span class="at"> json</span></span>
+<span id="cb1-4"><a href="#cb1-4" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">type</span><span class="kw">:</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
 </div>
+<p>Sample jsonl:</p>
+<pre class="jsonl"><code>{"input_ids":[271,299,99],"attention_mask":[1,1,1],"labels":[271,-100,99]}
+{"input_ids":[87,227,8383,12],"attention_mask":[1,1,1,1],"labels":[87,227,8383,12]}</code></pre>
 
 
 
diff --git a/docs/dataset_preprocessing.html b/docs/dataset_preprocessing.html
index 61726f0ff..c4ee4b37c 100644
--- a/docs/dataset_preprocessing.html
+++ b/docs/dataset_preprocessing.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
diff --git a/docs/debugging.html b/docs/debugging.html
index ba6074d54..08fd5fea9 100644
--- a/docs/debugging.html
+++ b/docs/debugging.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -34,7 +34,7 @@ div.sourceCode { overflow: auto; }
 }
 @media print {
 pre > code.sourceCode { white-space: pre-wrap; }
-pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
+pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
 }
 pre.numberSource code
   { counter-reset: source-line 0; }
diff --git a/docs/faq.html b/docs/faq.html
index cab6eb8c5..96942903e 100644
--- a/docs/faq.html
+++ b/docs/faq.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
diff --git a/docs/fsdp_qlora.html b/docs/fsdp_qlora.html
index 7c3903000..c87c446ee 100644
--- a/docs/fsdp_qlora.html
+++ b/docs/fsdp_qlora.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
diff --git a/docs/input_output.html b/docs/input_output.html
index bf3bee994..067dc11ef 100644
--- a/docs/input_output.html
+++ b/docs/input_output.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -34,7 +34,7 @@ div.sourceCode { overflow: auto; }
 }
 @media print {
 pre > code.sourceCode { white-space: pre-wrap; }
-pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
+pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
 }
 pre.numberSource code
   { counter-reset: source-line 0; }
diff --git a/docs/mac.html b/docs/mac.html
index 86e8c478a..e7398c12a 100644
--- a/docs/mac.html
+++ b/docs/mac.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
diff --git a/docs/multi-node.html b/docs/multi-node.html
index 3bdf34745..5f857da9f 100644
--- a/docs/multi-node.html
+++ b/docs/multi-node.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -34,7 +34,7 @@ div.sourceCode { overflow: auto; }
 }
 @media print {
 pre > code.sourceCode { white-space: pre-wrap; }
-pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
+pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
 }
 pre.numberSource code
   { counter-reset: source-line 0; }
diff --git a/docs/multipack.html b/docs/multipack.html
index 54e316302..43d1ce025 100644
--- a/docs/multipack.html
+++ b/docs/multipack.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
diff --git a/docs/nccl.html b/docs/nccl.html
index 85d6326d3..a150c7262 100644
--- a/docs/nccl.html
+++ b/docs/nccl.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
diff --git a/docs/rlhf.html b/docs/rlhf.html
index 0492db692..862c43c96 100644
--- a/docs/rlhf.html
+++ b/docs/rlhf.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -34,7 +34,7 @@ div.sourceCode { overflow: auto; }
 }
 @media print {
 pre > code.sourceCode { white-space: pre-wrap; }
-pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
+pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
 }
 pre.numberSource code
   { counter-reset: source-line 0; }
diff --git a/examples/colab-notebooks/colab-axolotl-example.html b/examples/colab-notebooks/colab-axolotl-example.html
index 82f020f4f..98497a12d 100644
--- a/examples/colab-notebooks/colab-axolotl-example.html
+++ b/examples/colab-notebooks/colab-axolotl-example.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -33,7 +33,7 @@ div.sourceCode { overflow: auto; }
 }
 @media print {
 pre > code.sourceCode { white-space: pre-wrap; }
-pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
+pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
 }
 pre.numberSource code
   { counter-reset: source-line 0; }
diff --git a/index.html b/index.html
index a5f7b8470..2cc7d213d 100644
--- a/index.html
+++ b/index.html
@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.4.555">
+<meta name="generator" content="quarto-1.4.557">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -33,7 +33,7 @@ div.sourceCode { overflow: auto; }
 }
 @media print {
 pre > code.sourceCode { white-space: pre-wrap; }
-pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
+pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
 }
 pre.numberSource code
   { counter-reset: source-line 0; }
diff --git a/search.json b/search.json
index 4d35e263d..b0249ea18 100644
--- a/search.json
+++ b/search.json
@@ -414,7 +414,7 @@
     "href": "docs/dataset-formats/tokenized.html",
     "title": "Custom Pre-Tokenized Dataset",
     "section": "",
-    "text": "Do not pass a type: in your axolotl config.\nColumns in Dataset must be exactly input_ids, attention_mask, labels\n\n\n\nconfig.yml\n\n- path: ...",
+    "text": "Pass an empty type: in your axolotl config.\nColumns in Dataset must be exactly input_ids, attention_mask, labels\nTo indicate that a token should be ignored during training, set its corresponding label to -100.\nDo not add BOS/EOS. Axolotl will add them for you based on the default tokenizer for the model you’re using.\nFor pretraining, do not truncate/pad documents to the context window length.\nFor instruction training, documents must be truncated/padded as desired.\n\nSample config:\n\n\nconfig.yml\n\ndatasets:\n  - path: /path/to/your/file.jsonl\n    ds_type: json\n    type:\n\nSample jsonl:\n{\"input_ids\":[271,299,99],\"attention_mask\":[1,1,1],\"labels\":[271,-100,99]}\n{\"input_ids\":[87,227,8383,12],\"attention_mask\":[1,1,1,1],\"labels\":[87,227,8383,12]}",
     "crumbs": [
       "Dataset Formats",
       "Custom Pre-Tokenized Dataset"
diff --git a/sitemap.xml b/sitemap.xml
index 5f1e267e0..6259d75de 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,90 +2,90 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/TODO.html</loc>
-    <lastmod>2024-06-20T14:05:28.768Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.824Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset_preprocessing.html</loc>
-    <lastmod>2024-06-20T14:05:28.772Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/config.html</loc>
-    <lastmod>2024-06-20T14:05:28.768Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.824Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/mac.html</loc>
-    <lastmod>2024-06-20T14:05:28.772Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset-formats/template_free.html</loc>
-    <lastmod>2024-06-20T14:05:28.768Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset-formats/index.html</loc>
-    <lastmod>2024-06-20T14:05:28.768Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.824Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset-formats/conversation.html</loc>
-    <lastmod>2024-06-20T14:05:28.768Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.824Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/debugging.html</loc>
-    <lastmod>2024-06-20T14:05:28.772Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/nccl.html</loc>
-    <lastmod>2024-06-20T14:05:28.772Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/rlhf.html</loc>
-    <lastmod>2024-06-20T14:05:28.772Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/examples/colab-notebooks/colab-axolotl-example.html</loc>
-    <lastmod>2024-06-20T14:05:28.772Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/FAQS.html</loc>
-    <lastmod>2024-06-20T14:05:28.768Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.824Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/index.html</loc>
-    <lastmod>2024-06-20T14:05:28.784Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.840Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/batch_vs_grad.html</loc>
-    <lastmod>2024-06-20T14:05:28.768Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.824Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/input_output.html</loc>
-    <lastmod>2024-06-20T14:05:28.772Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset-formats/tokenized.html</loc>
-    <lastmod>2024-06-20T14:05:28.768Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset-formats/pretraining.html</loc>
-    <lastmod>2024-06-20T14:05:28.768Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset-formats/inst_tune.html</loc>
-    <lastmod>2024-06-20T14:05:28.768Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/multi-node.html</loc>
-    <lastmod>2024-06-20T14:05:28.772Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/multipack.html</loc>
-    <lastmod>2024-06-20T14:05:28.772Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/fsdp_qlora.html</loc>
-    <lastmod>2024-06-20T14:05:28.772Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
   <url>
     <loc>https://OpenAccess-AI-Collective.github.io/axolotl/docs/faq.html</loc>
-    <lastmod>2024-06-20T14:05:28.772Z</lastmod>
+    <lastmod>2024-06-29T05:39:05.828Z</lastmod>
   </url>
 </urlset>
