Built site for gh-pages
This commit is contained in:
Quarto GHA Workflow Runner
730
docs/batch_vs_grad.html
Normal file
730
docs/batch_vs_grad.html
Normal file
@@ -0,0 +1,730 @@
|
||||
<!DOCTYPE html>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="generator" content="quarto-1.4.553">
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
|
||||
|
||||
<meta name="description" content="Understanding of batch size and gradient accumulation steps">
|
||||
|
||||
<title>Axolotl - Batch size vs Gradient accumulation</title>
|
||||
<style>
|
||||
code{white-space: pre-wrap;}
|
||||
span.smallcaps{font-variant: small-caps;}
|
||||
div.columns{display: flex; gap: min(4vw, 1.5em);}
|
||||
div.column{flex: auto; overflow-x: auto;}
|
||||
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
|
||||
ul.task-list{list-style: none;}
|
||||
ul.task-list li input[type="checkbox"] {
|
||||
width: 0.8em;
|
||||
margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
|
||||
vertical-align: middle;
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
|
||||
<script src="../site_libs/clipboard/clipboard.min.js"></script>
|
||||
<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
|
||||
<script src="../site_libs/quarto-search/fuse.min.js"></script>
|
||||
<script src="../site_libs/quarto-search/quarto-search.js"></script>
|
||||
<meta name="quarto:offset" content="../">
|
||||
<link href="../favicon.jpg" rel="icon" type="image/jpeg">
|
||||
<script src="../site_libs/quarto-html/quarto.js"></script>
|
||||
<script src="../site_libs/quarto-html/popper.min.js"></script>
|
||||
<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
|
||||
<script src="../site_libs/quarto-html/anchor.min.js"></script>
|
||||
<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
|
||||
<link href="../site_libs/quarto-html/quarto-syntax-highlighting.css" rel="stylesheet" id="quarto-text-highlighting-styles">
|
||||
<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
|
||||
<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
|
||||
<link href="../site_libs/bootstrap/bootstrap.min.css" rel="stylesheet" id="quarto-bootstrap" data-mode="light">
|
||||
<script id="quarto-search-options" type="application/json">{
|
||||
"location": "navbar",
|
||||
"copy-button": false,
|
||||
"collapse-after": 3,
|
||||
"panel-placement": "end",
|
||||
"type": "overlay",
|
||||
"limit": 50,
|
||||
"keyboard-shortcut": [
|
||||
"f",
|
||||
"/",
|
||||
"s"
|
||||
],
|
||||
"show-item-context": false,
|
||||
"language": {
|
||||
"search-no-results-text": "No results",
|
||||
"search-matching-documents-text": "matching documents",
|
||||
"search-copy-link-title": "Copy link to search",
|
||||
"search-hide-matches-text": "Hide additional matches",
|
||||
"search-more-match-text": "more match in this document",
|
||||
"search-more-matches-text": "more matches in this document",
|
||||
"search-clear-button-title": "Clear",
|
||||
"search-text-placeholder": "",
|
||||
"search-detached-cancel-button-title": "Cancel",
|
||||
"search-submit-button-title": "Submit",
|
||||
"search-label": "Search"
|
||||
}
|
||||
}</script>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../styles.css">
|
||||
</head>
|
||||
|
||||
<body class="nav-sidebar docked nav-fixed">
|
||||
|
||||
<div id="quarto-search-results"></div>
|
||||
<header id="quarto-header" class="headroom fixed-top">
|
||||
<nav class="navbar navbar-expand " data-bs-theme="dark">
|
||||
<div class="navbar-container container-fluid">
|
||||
<div class="navbar-brand-container mx-auto">
|
||||
<a class="navbar-brand" href="../index.html">
|
||||
<span class="navbar-title">Axolotl</span>
|
||||
</a>
|
||||
</div>
|
||||
<div class="quarto-navbar-tools tools-wide tools-end">
|
||||
<a href="https://twitter.com/axolotl_ai" title="" class="quarto-navigation-tool px-1" aria-label=""><i class="bi bi-twitter"></i></a>
|
||||
<a href="https://github.com/OpenAccess-AI-Collective/axolotl/" title="" class="quarto-navigation-tool px-1" aria-label=""><i class="bi bi-github"></i></a>
|
||||
<a href="https://discord.gg/7m9sfhzaf3" title="" class="quarto-navigation-tool px-1" aria-label=""><i class="bi bi-discord"></i></a>
|
||||
</div>
|
||||
<div id="quarto-search" class="" title="Search"></div>
|
||||
</div> <!-- /container-fluid -->
|
||||
</nav>
|
||||
<nav class="quarto-secondary-nav">
|
||||
<div class="container-fluid d-flex">
|
||||
<button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
|
||||
<i class="bi bi-layout-text-sidebar-reverse"></i>
|
||||
</button>
|
||||
<nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item">Batch size vs Gradient accumulation</li></ol></nav>
|
||||
<a class="flex-grow-1" role="button" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
|
||||
</a>
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
<!-- content -->
|
||||
<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
|
||||
<!-- sidebar -->
|
||||
<nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation docked overflow-auto">
|
||||
<div class="sidebar-menu-container">
|
||||
<ul class="list-unstyled mt-1">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../index.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Home</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item sidebar-item-section">
|
||||
<div class="sidebar-item-container">
|
||||
<a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" aria-expanded="true">
|
||||
<span class="menu-text">How-To Guides</span></a>
|
||||
<a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" aria-expanded="true" aria-label="Toggle section">
|
||||
<i class="bi bi-chevron-right ms-2"></i>
|
||||
</a>
|
||||
</div>
|
||||
<ul id="quarto-sidebar-section-1" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/debugging.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Debugging</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/multipack.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Multipack (Sample Packing)</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/fsdp_qlora.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">FDSP + QLoRA</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/input_output.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Template-free prompt construction</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/rlhf.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">RLHF (Beta)</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/nccl.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">NCCL</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/mac.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Mac M-series</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/multi-node.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Multi Node</span></a>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="sidebar-item sidebar-item-section">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/index.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Dataset Formats</span></a>
|
||||
<a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" aria-expanded="true" aria-label="Toggle section">
|
||||
<i class="bi bi-chevron-right ms-2"></i>
|
||||
</a>
|
||||
</div>
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/inst_tune.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Instruction Tuning</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/template_free.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Template-Free</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/tokenized.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Custom Pre-Tokenized Dataset</span></a>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="sidebar-item sidebar-item-section">
|
||||
<div class="sidebar-item-container">
|
||||
<a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-3" aria-expanded="true">
|
||||
<span class="menu-text">Reference</span></a>
|
||||
<a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-3" aria-expanded="true" aria-label="Toggle section">
|
||||
<i class="bi bi-chevron-right ms-2"></i>
|
||||
</a>
|
||||
</div>
|
||||
<ul id="quarto-sidebar-section-3" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/config.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Config options</span></a>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/faq.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">FAQ</span></a>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</nav>
|
||||
<div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div>
|
||||
<!-- margin-sidebar -->
|
||||
<div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
|
||||
|
||||
</div>
|
||||
<!-- main -->
|
||||
<main class="content" id="quarto-document-content">
|
||||
|
||||
<header id="title-block-header" class="quarto-title-block default">
|
||||
<div class="quarto-title">
|
||||
<h1 class="title">Batch size vs Gradient accumulation</h1>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<div class="description">
|
||||
Understanding of batch size and gradient accumulation steps
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="quarto-title-meta">
|
||||
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
</header>
|
||||
|
||||
|
||||
<p>Gradient accumulation means accumulating gradients over several mini-batches and updating the model weights afterward. When the samples in each batch are diverse, this technique doesn’t significantly impact learning.</p>
|
||||
<p>This method allows for effective training with larger effective batch sizes without needing proportionally larger memory. Here’s why:</p>
|
||||
<ol type="1">
|
||||
<li><p><strong>Memory Consumption with Batch Size</strong>: The primary reason increasing the batch size impacts memory is due to the storage requirements for intermediate activations. When you forward propagate a batch through a network, you have to store the activations at each layer for each sample in the batch, because these activations are used during backpropagation to compute gradients. Therefore, larger batches mean more activations, leading to greater GPU memory consumption.</p></li>
|
||||
<li><p><strong>Gradient Accumulation</strong>: With gradient accumulation, you’re effectively simulating a larger batch size by accumulating gradients over several smaller batches (or micro-batches). However, at any given time, you’re only forward and backward propagating a micro-batch. This means you only store activations for the micro-batch, not the full accumulated batch. As a result, you can simulate the effect of a larger batch size without the memory cost of storing activations for a large batch.</p></li>
|
||||
</ol>
|
||||
<p><strong>Example 1:</strong> Micro batch size: 3 Gradient accumulation steps: 2 Number of GPUs: 3 Total batch size = 3 * 2 * 3 = 18</p>
|
||||
<pre><code>| GPU 1 | GPU 2 | GPU 3 |
|
||||
|----------------|----------------|----------------|
|
||||
| S1, S2, S3 | S4, S5, S6 | S7, S8, S9 |
|
||||
| e1, e2, e3 | e4, e5, e6 | e7, e8, e9 |
|
||||
|----------------|----------------|----------------|
|
||||
| → (accumulate) | → (accumulate) | → (accumulate) |
|
||||
|----------------|----------------|----------------|
|
||||
| S10, S11, S12 | S13, S14, S15 | S16, S17, S18 |
|
||||
| e10, e11, e12 | e13, e14, e15 | e16, e17, e18 |
|
||||
|----------------|----------------|----------------|
|
||||
| → (apply) | → (apply) | → (apply) |
|
||||
|
||||
Accumulated gradient for the weight w1 after the second iteration (considering all GPUs):
|
||||
Total gradient for w1 = e1 + e2 + e3 + e4 + e5 + e6 + e7 + e8 + e9 + e10 + e11 + e12 + e13 + e14 + e15 + e16 + e17 + e18
|
||||
|
||||
Weight update for w1:
|
||||
w1_new = w1_old - learning rate x (Total gradient for w1 / 18)</code></pre>
|
||||
<p><strong>Example 2:</strong> Micro batch size: 2 Gradient accumulation steps: 1 Number of GPUs: 3 Total batch size = 2 * 1 * 3 = 6</p>
|
||||
<pre><code>| GPU 1 | GPU 2 | GPU 3 |
|
||||
|-----------|-----------|-----------|
|
||||
| S1, S2 | S3, S4 | S5, S6 |
|
||||
| e1, e2 | e3, e4 | e5, e6 |
|
||||
|-----------|-----------|-----------|
|
||||
| → (apply) | → (apply) | → (apply) |
|
||||
|
||||
Accumulated gradient for the weight w1 (considering all GPUs):
|
||||
Total gradient for w1 = e1 + e2 + e3 + e4 + e5 + e6
|
||||
|
||||
Weight update for w1:
|
||||
w1_new = w1_old - learning rate × (Total gradient for w1 / 6)</code></pre>
|
||||
|
||||
|
||||
|
||||
</main> <!-- /main -->
|
||||
<script id="quarto-html-after-body" type="application/javascript">
|
||||
window.document.addEventListener("DOMContentLoaded", function (event) {
|
||||
const toggleBodyColorMode = (bsSheetEl) => {
|
||||
const mode = bsSheetEl.getAttribute("data-mode");
|
||||
const bodyEl = window.document.querySelector("body");
|
||||
if (mode === "dark") {
|
||||
bodyEl.classList.add("quarto-dark");
|
||||
bodyEl.classList.remove("quarto-light");
|
||||
} else {
|
||||
bodyEl.classList.add("quarto-light");
|
||||
bodyEl.classList.remove("quarto-dark");
|
||||
}
|
||||
}
|
||||
const toggleBodyColorPrimary = () => {
|
||||
const bsSheetEl = window.document.querySelector("link#quarto-bootstrap");
|
||||
if (bsSheetEl) {
|
||||
toggleBodyColorMode(bsSheetEl);
|
||||
}
|
||||
}
|
||||
toggleBodyColorPrimary();
|
||||
const icon = "";
|
||||
const anchorJS = new window.AnchorJS();
|
||||
anchorJS.options = {
|
||||
placement: 'right',
|
||||
icon: icon
|
||||
};
|
||||
anchorJS.add('.anchored');
|
||||
const isCodeAnnotation = (el) => {
|
||||
for (const clz of el.classList) {
|
||||
if (clz.startsWith('code-annotation-')) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
const clipboard = new window.ClipboardJS('.code-copy-button', {
|
||||
text: function(trigger) {
|
||||
const codeEl = trigger.previousElementSibling.cloneNode(true);
|
||||
for (const childEl of codeEl.children) {
|
||||
if (isCodeAnnotation(childEl)) {
|
||||
childEl.remove();
|
||||
}
|
||||
}
|
||||
return codeEl.innerText;
|
||||
}
|
||||
});
|
||||
clipboard.on('success', function(e) {
|
||||
// button target
|
||||
const button = e.trigger;
|
||||
// don't keep focus
|
||||
button.blur();
|
||||
// flash "checked"
|
||||
button.classList.add('code-copy-button-checked');
|
||||
var currentTitle = button.getAttribute("title");
|
||||
button.setAttribute("title", "Copied!");
|
||||
let tooltip;
|
||||
if (window.bootstrap) {
|
||||
button.setAttribute("data-bs-toggle", "tooltip");
|
||||
button.setAttribute("data-bs-placement", "left");
|
||||
button.setAttribute("data-bs-title", "Copied!");
|
||||
tooltip = new bootstrap.Tooltip(button,
|
||||
{ trigger: "manual",
|
||||
customClass: "code-copy-button-tooltip",
|
||||
offset: [0, -8]});
|
||||
tooltip.show();
|
||||
}
|
||||
setTimeout(function() {
|
||||
if (tooltip) {
|
||||
tooltip.hide();
|
||||
button.removeAttribute("data-bs-title");
|
||||
button.removeAttribute("data-bs-toggle");
|
||||
button.removeAttribute("data-bs-placement");
|
||||
}
|
||||
button.setAttribute("title", currentTitle);
|
||||
button.classList.remove('code-copy-button-checked');
|
||||
}, 1000);
|
||||
// clear code selection
|
||||
e.clearSelection();
|
||||
});
|
||||
var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
|
||||
var mailtoRegex = new RegExp(/^mailto:/);
|
||||
var filterRegex = new RegExp("https:\/\/OpenAccess-AI-Collective\.github\.io\/axolotl\/");
|
||||
var isInternal = (href) => {
|
||||
return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
|
||||
}
|
||||
// Inspect non-navigation links and adorn them if external
|
||||
var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool)');
|
||||
for (var i=0; i<links.length; i++) {
|
||||
const link = links[i];
|
||||
if (!isInternal(link.href)) {
|
||||
// undo the damage that might have been done by quarto-nav.js in the case of
|
||||
// links that we want to consider external
|
||||
if (link.dataset.originalHref !== undefined) {
|
||||
link.href = link.dataset.originalHref;
|
||||
}
|
||||
}
|
||||
}
|
||||
function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
|
||||
const config = {
|
||||
allowHTML: true,
|
||||
maxWidth: 500,
|
||||
delay: 100,
|
||||
arrow: false,
|
||||
appendTo: function(el) {
|
||||
return el.parentElement;
|
||||
},
|
||||
interactive: true,
|
||||
interactiveBorder: 10,
|
||||
theme: 'quarto',
|
||||
placement: 'bottom-start',
|
||||
};
|
||||
if (contentFn) {
|
||||
config.content = contentFn;
|
||||
}
|
||||
if (onTriggerFn) {
|
||||
config.onTrigger = onTriggerFn;
|
||||
}
|
||||
if (onUntriggerFn) {
|
||||
config.onUntrigger = onUntriggerFn;
|
||||
}
|
||||
window.tippy(el, config);
|
||||
}
|
||||
const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
|
||||
for (var i=0; i<noterefs.length; i++) {
|
||||
const ref = noterefs[i];
|
||||
tippyHover(ref, function() {
|
||||
// use id or data attribute instead here
|
||||
let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
|
||||
try { href = new URL(href).hash; } catch {}
|
||||
const id = href.replace(/^#\/?/, "");
|
||||
const note = window.document.getElementById(id);
|
||||
if (note) {
|
||||
return note.innerHTML;
|
||||
} else {
|
||||
return "";
|
||||
}
|
||||
});
|
||||
}
|
||||
const xrefs = window.document.querySelectorAll('a.quarto-xref');
|
||||
const processXRef = (id, note) => {
|
||||
// Strip column container classes
|
||||
const stripColumnClz = (el) => {
|
||||
el.classList.remove("page-full", "page-columns");
|
||||
if (el.children) {
|
||||
for (const child of el.children) {
|
||||
stripColumnClz(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
stripColumnClz(note)
|
||||
if (id === null || id.startsWith('sec-')) {
|
||||
// Special case sections, only their first couple elements
|
||||
const container = document.createElement("div");
|
||||
if (note.children && note.children.length > 2) {
|
||||
container.appendChild(note.children[0].cloneNode(true));
|
||||
for (let i = 1; i < note.children.length; i++) {
|
||||
const child = note.children[i];
|
||||
if (child.tagName === "P" && child.innerText === "") {
|
||||
continue;
|
||||
} else {
|
||||
container.appendChild(child.cloneNode(true));
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (window.Quarto?.typesetMath) {
|
||||
window.Quarto.typesetMath(container);
|
||||
}
|
||||
return container.innerHTML
|
||||
} else {
|
||||
if (window.Quarto?.typesetMath) {
|
||||
window.Quarto.typesetMath(note);
|
||||
}
|
||||
return note.innerHTML;
|
||||
}
|
||||
} else {
|
||||
// Remove any anchor links if they are present
|
||||
const anchorLink = note.querySelector('a.anchorjs-link');
|
||||
if (anchorLink) {
|
||||
anchorLink.remove();
|
||||
}
|
||||
if (window.Quarto?.typesetMath) {
|
||||
window.Quarto.typesetMath(note);
|
||||
}
|
||||
// TODO in 1.5, we should make sure this works without a callout special case
|
||||
if (note.classList.contains("callout")) {
|
||||
return note.outerHTML;
|
||||
} else {
|
||||
return note.innerHTML;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (var i=0; i<xrefs.length; i++) {
|
||||
const xref = xrefs[i];
|
||||
tippyHover(xref, undefined, function(instance) {
|
||||
instance.disable();
|
||||
let url = xref.getAttribute('href');
|
||||
let hash = undefined;
|
||||
if (url.startsWith('#')) {
|
||||
hash = url;
|
||||
} else {
|
||||
try { hash = new URL(url).hash; } catch {}
|
||||
}
|
||||
if (hash) {
|
||||
const id = hash.replace(/^#\/?/, "");
|
||||
const note = window.document.getElementById(id);
|
||||
if (note !== null) {
|
||||
try {
|
||||
const html = processXRef(id, note.cloneNode(true));
|
||||
instance.setContent(html);
|
||||
} finally {
|
||||
instance.enable();
|
||||
instance.show();
|
||||
}
|
||||
} else {
|
||||
// See if we can fetch this
|
||||
fetch(url.split('#')[0])
|
||||
.then(res => res.text())
|
||||
.then(html => {
|
||||
const parser = new DOMParser();
|
||||
const htmlDoc = parser.parseFromString(html, "text/html");
|
||||
const note = htmlDoc.getElementById(id);
|
||||
if (note !== null) {
|
||||
const html = processXRef(id, note);
|
||||
instance.setContent(html);
|
||||
}
|
||||
}).finally(() => {
|
||||
instance.enable();
|
||||
instance.show();
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// See if we can fetch a full url (with no hash to target)
|
||||
// This is a special case and we should probably do some content thinning / targeting
|
||||
fetch(url)
|
||||
.then(res => res.text())
|
||||
.then(html => {
|
||||
const parser = new DOMParser();
|
||||
const htmlDoc = parser.parseFromString(html, "text/html");
|
||||
const note = htmlDoc.querySelector('main.content');
|
||||
if (note !== null) {
|
||||
// This should only happen for chapter cross references
|
||||
// (since there is no id in the URL)
|
||||
// remove the first header
|
||||
if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
|
||||
note.children[0].remove();
|
||||
}
|
||||
const html = processXRef(null, note);
|
||||
instance.setContent(html);
|
||||
}
|
||||
}).finally(() => {
|
||||
instance.enable();
|
||||
instance.show();
|
||||
});
|
||||
}
|
||||
}, function(instance) {
|
||||
});
|
||||
}
|
||||
let selectedAnnoteEl;
|
||||
const selectorForAnnotation = ( cell, annotation) => {
|
||||
let cellAttr = 'data-code-cell="' + cell + '"';
|
||||
let lineAttr = 'data-code-annotation="' + annotation + '"';
|
||||
const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
|
||||
return selector;
|
||||
}
|
||||
const selectCodeLines = (annoteEl) => {
|
||||
const doc = window.document;
|
||||
const targetCell = annoteEl.getAttribute("data-target-cell");
|
||||
const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
|
||||
const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
|
||||
const lines = annoteSpan.getAttribute("data-code-lines").split(",");
|
||||
const lineIds = lines.map((line) => {
|
||||
return targetCell + "-" + line;
|
||||
})
|
||||
let top = null;
|
||||
let height = null;
|
||||
let parent = null;
|
||||
if (lineIds.length > 0) {
|
||||
//compute the position of the single el (top and bottom and make a div)
|
||||
const el = window.document.getElementById(lineIds[0]);
|
||||
top = el.offsetTop;
|
||||
height = el.offsetHeight;
|
||||
parent = el.parentElement.parentElement;
|
||||
if (lineIds.length > 1) {
|
||||
const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
|
||||
const bottom = lastEl.offsetTop + lastEl.offsetHeight;
|
||||
height = bottom - top;
|
||||
}
|
||||
if (top !== null && height !== null && parent !== null) {
|
||||
// cook up a div (if necessary) and position it
|
||||
let div = window.document.getElementById("code-annotation-line-highlight");
|
||||
if (div === null) {
|
||||
div = window.document.createElement("div");
|
||||
div.setAttribute("id", "code-annotation-line-highlight");
|
||||
div.style.position = 'absolute';
|
||||
parent.appendChild(div);
|
||||
}
|
||||
div.style.top = top - 2 + "px";
|
||||
div.style.height = height + 4 + "px";
|
||||
div.style.left = 0;
|
||||
let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
|
||||
if (gutterDiv === null) {
|
||||
gutterDiv = window.document.createElement("div");
|
||||
gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
|
||||
gutterDiv.style.position = 'absolute';
|
||||
const codeCell = window.document.getElementById(targetCell);
|
||||
const gutter = codeCell.querySelector('.code-annotation-gutter');
|
||||
gutter.appendChild(gutterDiv);
|
||||
}
|
||||
gutterDiv.style.top = top - 2 + "px";
|
||||
gutterDiv.style.height = height + 4 + "px";
|
||||
}
|
||||
selectedAnnoteEl = annoteEl;
|
||||
}
|
||||
};
|
||||
const unselectCodeLines = () => {
|
||||
const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
|
||||
elementsIds.forEach((elId) => {
|
||||
const div = window.document.getElementById(elId);
|
||||
if (div) {
|
||||
div.remove();
|
||||
}
|
||||
});
|
||||
selectedAnnoteEl = undefined;
|
||||
};
|
||||
// Handle positioning of the toggle
|
||||
window.addEventListener(
|
||||
"resize",
|
||||
throttle(() => {
|
||||
elRect = undefined;
|
||||
if (selectedAnnoteEl) {
|
||||
selectCodeLines(selectedAnnoteEl);
|
||||
}
|
||||
}, 10)
|
||||
);
|
||||
function throttle(fn, ms) {
|
||||
let throttle = false;
|
||||
let timer;
|
||||
return (...args) => {
|
||||
if(!throttle) { // first call gets through
|
||||
fn.apply(this, args);
|
||||
throttle = true;
|
||||
} else { // all the others get throttled
|
||||
if(timer) clearTimeout(timer); // cancel #2
|
||||
timer = setTimeout(() => {
|
||||
fn.apply(this, args);
|
||||
timer = throttle = false;
|
||||
}, ms);
|
||||
}
|
||||
};
|
||||
}
|
||||
// Attach click handler to the DT
|
||||
const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
|
||||
for (const annoteDlNode of annoteDls) {
|
||||
annoteDlNode.addEventListener('click', (event) => {
|
||||
const clickedEl = event.target;
|
||||
if (clickedEl !== selectedAnnoteEl) {
|
||||
unselectCodeLines();
|
||||
const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
|
||||
if (activeEl) {
|
||||
activeEl.classList.remove('code-annotation-active');
|
||||
}
|
||||
selectCodeLines(clickedEl);
|
||||
clickedEl.classList.add('code-annotation-active');
|
||||
} else {
|
||||
// Unselect the line
|
||||
unselectCodeLines();
|
||||
clickedEl.classList.remove('code-annotation-active');
|
||||
}
|
||||
});
|
||||
}
|
||||
const findCites = (el) => {
|
||||
const parentEl = el.parentElement;
|
||||
if (parentEl) {
|
||||
const cites = parentEl.dataset.cites;
|
||||
if (cites) {
|
||||
return {
|
||||
el,
|
||||
cites: cites.split(' ')
|
||||
};
|
||||
} else {
|
||||
return findCites(el.parentElement)
|
||||
}
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
|
||||
for (var i=0; i<bibliorefs.length; i++) {
|
||||
const ref = bibliorefs[i];
|
||||
const citeInfo = findCites(ref);
|
||||
if (citeInfo) {
|
||||
tippyHover(citeInfo.el, function() {
|
||||
var popup = window.document.createElement('div');
|
||||
citeInfo.cites.forEach(function(cite) {
|
||||
var citeDiv = window.document.createElement('div');
|
||||
citeDiv.classList.add('hanging-indent');
|
||||
citeDiv.classList.add('csl-entry');
|
||||
var biblioDiv = window.document.getElementById('ref-' + cite);
|
||||
if (biblioDiv) {
|
||||
citeDiv.innerHTML = biblioDiv.innerHTML;
|
||||
}
|
||||
popup.appendChild(citeDiv);
|
||||
});
|
||||
return popup.innerHTML;
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</div> <!-- /content -->
|
||||
|
||||
|
||||
|
||||
|
||||
</body></html>
|
||||
@@ -218,8 +218,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -230,8 +230,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
|
||||
@@ -218,8 +218,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link active">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -230,8 +230,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link active">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -281,15 +281,11 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
<h2 id="toc-title">On this page</h2>
|
||||
|
||||
<ul>
|
||||
<li><a href="#formats" id="toc-formats" class="nav-link active" data-scroll-target="#formats">Formats</a>
|
||||
<ul class="collapse">
|
||||
<li><a href="#sharegpt" id="toc-sharegpt" class="nav-link" data-scroll-target="#sharegpt">sharegpt</a></li>
|
||||
<li><a href="#sharegpt" id="toc-sharegpt" class="nav-link active" data-scroll-target="#sharegpt">sharegpt</a></li>
|
||||
<li><a href="#pygmalion" id="toc-pygmalion" class="nav-link" data-scroll-target="#pygmalion">pygmalion</a></li>
|
||||
<li><a href="#sharegpt.load_role" id="toc-sharegpt.load_role" class="nav-link" data-scroll-target="#sharegpt.load_role">sharegpt.load_role</a></li>
|
||||
<li><a href="#sharegpt.load_guanaco" id="toc-sharegpt.load_guanaco" class="nav-link" data-scroll-target="#sharegpt.load_guanaco">sharegpt.load_guanaco</a></li>
|
||||
<li><a href="#sharegpt_jokes" id="toc-sharegpt_jokes" class="nav-link" data-scroll-target="#sharegpt_jokes">sharegpt_jokes</a></li>
|
||||
</ul></li>
|
||||
<li><a href="#how-to-add-custom-prompts-for-instruction-tuning" id="toc-how-to-add-custom-prompts-for-instruction-tuning" class="nav-link" data-scroll-target="#how-to-add-custom-prompts-for-instruction-tuning">How to add custom prompts for instruction-tuning</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
@@ -320,10 +316,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</header>
|
||||
|
||||
|
||||
<section id="formats" class="level2">
|
||||
<h2 class="anchored" data-anchor-id="formats">Formats</h2>
|
||||
<section id="sharegpt" class="level3">
|
||||
<h3 class="anchored" data-anchor-id="sharegpt">sharegpt</h3>
|
||||
<section id="sharegpt" class="level2">
|
||||
<h2 class="anchored" data-anchor-id="sharegpt">sharegpt</h2>
|
||||
<p>conversations where <code>from</code> is <code>human</code>/<code>gpt</code>. (optional: first row with role <code>system</code> to override default system prompt)</p>
|
||||
<div class="code-with-filename">
|
||||
<div class="code-with-filename-file">
|
||||
@@ -331,20 +325,21 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</div>
|
||||
<div class="sourceCode" id="cb1" data-filename="data.jsonl"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span><span class="dt">"conversations"</span><span class="fu">:</span> <span class="ot">[</span><span class="fu">{</span><span class="dt">"from"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"value"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">}</span><span class="ot">]</span><span class="fu">}</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
</div>
|
||||
<p>Note: <code>type: sharegpt</code> opens a special config <code>conversation:</code> that enables conversions to many Conversation types. See <a href="../docs/config.qmd">the docs</a> for all config options.</p>
|
||||
<p>Note: <code>type: sharegpt</code> opens special configs: - <code>conversation</code>: enables conversions to many Conversation types. Refer to the ‘name’ <a href="https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py">here</a> for options. - <code>roles</code>: allows you to specify the roles for input and output. This is useful for datasets with custom roles such as <code>tool</code> etc to support masking. - <code>field_human</code>: specify the key to use instead of <code>human</code> in the conversation. - <code>field_model</code>: specify the key to use instead of <code>gpt</code> in the conversation.</p>
|
||||
<div class="sourceCode" id="cb2"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="fu">datasets</span><span class="kw">:</span></span>
|
||||
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">path</span><span class="kw">:</span><span class="at"> ...</span></span>
|
||||
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">type</span><span class="kw">:</span><span class="at"> sharegpt</span></span>
|
||||
<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a></span>
|
||||
<span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">conversation</span><span class="kw">:</span><span class="co"> # Options (see Conversation 'name'): https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py</span></span>
|
||||
<span id="cb2-6"><a href="#cb2-6" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">field_human</span><span class="kw">:</span><span class="co"> # Optional[str]. Human key to use for conversation.</span></span>
|
||||
<span id="cb2-7"><a href="#cb2-7" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">field_model</span><span class="kw">:</span><span class="co"> # Optional[str]. Assistant key to use for conversation.</span></span>
|
||||
<span id="cb2-8"><a href="#cb2-8" aria-hidden="true" tabindex="-1"></a><span class="co"> # Add additional keys from your dataset as input or output roles</span></span>
|
||||
<span id="cb2-9"><a href="#cb2-9" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">roles</span><span class="kw">:</span></span>
|
||||
<span id="cb2-10"><a href="#cb2-10" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">input</span><span class="kw">:</span><span class="co"> # Optional[List[str]]. These will be masked based on train_on_input</span></span>
|
||||
<span id="cb2-11"><a href="#cb2-11" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">output</span><span class="kw">:</span><span class="co"> # Optional[List[str]].</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
</section>
|
||||
<section id="pygmalion" class="level3">
|
||||
<h3 class="anchored" data-anchor-id="pygmalion">pygmalion</h3>
|
||||
<div class="code-with-filename">
|
||||
<div class="code-with-filename-file">
|
||||
<pre><strong>data.jsonl</strong></pre>
|
||||
</div>
|
||||
<div class="sourceCode" id="cb2" data-filename="data.jsonl"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span><span class="dt">"conversations"</span><span class="fu">:</span> <span class="ot">[</span><span class="fu">{</span><span class="dt">"role"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"value"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">}</span><span class="ot">]</span><span class="fu">}</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="sharegpt.load_role" class="level3">
|
||||
<h3 class="anchored" data-anchor-id="sharegpt.load_role">sharegpt.load_role</h3>
|
||||
<p>conversations where <code>role</code> is used instead of <code>from</code></p>
|
||||
<section id="pygmalion" class="level2">
|
||||
<h2 class="anchored" data-anchor-id="pygmalion">pygmalion</h2>
|
||||
<div class="code-with-filename">
|
||||
<div class="code-with-filename-file">
|
||||
<pre><strong>data.jsonl</strong></pre>
|
||||
@@ -352,52 +347,35 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
<div class="sourceCode" id="cb3" data-filename="data.jsonl"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span><span class="dt">"conversations"</span><span class="fu">:</span> <span class="ot">[</span><span class="fu">{</span><span class="dt">"role"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"value"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">}</span><span class="ot">]</span><span class="fu">}</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="sharegpt.load_guanaco" class="level3">
|
||||
<h3 class="anchored" data-anchor-id="sharegpt.load_guanaco">sharegpt.load_guanaco</h3>
|
||||
<section id="sharegpt.load_role" class="level2">
|
||||
<h2 class="anchored" data-anchor-id="sharegpt.load_role">sharegpt.load_role</h2>
|
||||
<p>conversations where <code>role</code> is used instead of <code>from</code></p>
|
||||
<div class="code-with-filename">
|
||||
<div class="code-with-filename-file">
|
||||
<pre><strong>data.jsonl</strong></pre>
|
||||
</div>
|
||||
<div class="sourceCode" id="cb4" data-filename="data.jsonl"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span><span class="dt">"conversations"</span><span class="fu">:</span> <span class="ot">[</span><span class="fu">{</span><span class="dt">"role"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"value"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">}</span><span class="ot">]</span><span class="fu">}</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="sharegpt.load_guanaco" class="level2">
|
||||
<h2 class="anchored" data-anchor-id="sharegpt.load_guanaco">sharegpt.load_guanaco</h2>
|
||||
<p>conversations where <code>from</code> is <code>prompter</code> <code>assistant</code> instead of default sharegpt</p>
|
||||
<div class="code-with-filename">
|
||||
<div class="code-with-filename-file">
|
||||
<pre><strong>data.jsonl</strong></pre>
|
||||
</div>
|
||||
<div class="sourceCode" id="cb4" data-filename="data.jsonl"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span><span class="dt">"conversations"</span><span class="fu">:</span> <span class="ot">[</span><span class="fu">{</span><span class="dt">"from"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"value"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">}</span><span class="ot">]</span><span class="fu">}</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
<div class="sourceCode" id="cb5" data-filename="data.jsonl"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span><span class="dt">"conversations"</span><span class="fu">:</span> <span class="ot">[</span><span class="fu">{</span><span class="dt">"from"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"value"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">}</span><span class="ot">]</span><span class="fu">}</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="sharegpt_jokes" class="level3">
|
||||
<h3 class="anchored" data-anchor-id="sharegpt_jokes">sharegpt_jokes</h3>
|
||||
<section id="sharegpt_jokes" class="level2">
|
||||
<h2 class="anchored" data-anchor-id="sharegpt_jokes">sharegpt_jokes</h2>
|
||||
<p>creates a chat where bot is asked to tell a joke, then explain why the joke is funny</p>
|
||||
<div class="code-with-filename">
|
||||
<div class="code-with-filename-file">
|
||||
<pre><strong>data.jsonl</strong></pre>
|
||||
</div>
|
||||
<div class="sourceCode" id="cb5" data-filename="data.jsonl"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span><span class="dt">"conversations"</span><span class="fu">:</span> <span class="ot">[</span><span class="fu">{</span><span class="dt">"title"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"text"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"explanation"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">}</span><span class="ot">]</span><span class="fu">}</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
<div class="sourceCode" id="cb6" data-filename="data.jsonl"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span><span class="dt">"conversations"</span><span class="fu">:</span> <span class="ot">[</span><span class="fu">{</span><span class="dt">"title"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"text"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"explanation"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">}</span><span class="ot">]</span><span class="fu">}</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section id="how-to-add-custom-prompts-for-instruction-tuning" class="level2">
|
||||
<h2 class="anchored" data-anchor-id="how-to-add-custom-prompts-for-instruction-tuning">How to add custom prompts for instruction-tuning</h2>
|
||||
<p>For a dataset that is preprocessed for instruction purposes:</p>
|
||||
<div class="code-with-filename">
|
||||
<div class="code-with-filename-file">
|
||||
<pre><strong>data.jsonl</strong></pre>
|
||||
</div>
|
||||
<div class="sourceCode" id="cb6" data-filename="data.jsonl"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span><span class="dt">"input"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"output"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">}</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
</div>
|
||||
<p>You can use this example in your YAML config:</p>
|
||||
<div class="code-with-filename">
|
||||
<div class="code-with-filename-file">
|
||||
<pre><strong>config.yaml</strong></pre>
|
||||
</div>
|
||||
<div class="sourceCode" id="cb7" data-filename="config.yaml"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="fu">datasets</span><span class="kw">:</span></span>
|
||||
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="kw">-</span><span class="at"> </span><span class="fu">path</span><span class="kw">:</span><span class="at"> repo</span></span>
|
||||
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">type</span><span class="kw">:</span></span>
|
||||
<span id="cb7-4"><a href="#cb7-4" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">system_prompt</span><span class="kw">:</span><span class="at"> </span><span class="st">""</span></span>
|
||||
<span id="cb7-5"><a href="#cb7-5" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">field_system</span><span class="kw">:</span><span class="at"> system</span></span>
|
||||
<span id="cb7-6"><a href="#cb7-6" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">field_instruction</span><span class="kw">:</span><span class="at"> input</span></span>
|
||||
<span id="cb7-7"><a href="#cb7-7" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">field_output</span><span class="kw">:</span><span class="at"> output</span></span>
|
||||
<span id="cb7-8"><a href="#cb7-8" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">format</span><span class="kw">:</span><span class="at"> </span><span class="st">"[INST] {instruction} [/INST]"</span></span>
|
||||
<span id="cb7-9"><a href="#cb7-9" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">no_input_format</span><span class="kw">:</span><span class="at"> </span><span class="st">"[INST] {instruction} [/INST]"</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
</div>
|
||||
<p>See full config options under <a href="../docs/config.qmd">here</a>.</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
@@ -244,8 +244,8 @@ window.Quarto = {
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -256,8 +256,8 @@ window.Quarto = {
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -351,23 +351,7 @@ Description
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="list">
|
||||
<tr data-index="0" data-listing-file-modified-sort="1712171163568" data-listing-reading-time-sort="1" data-listing-word-count-sort="189" data-listing-title-sort="Conversation" data-listing-filename-sort="conversation.qmd">
|
||||
<td>
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="title listing-title">Conversation</a>
|
||||
</td>
|
||||
<td>
|
||||
<span class="listing-description">Conversation format for supervised fine-tuning.</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr data-index="1" data-listing-file-modified-sort="1712171163568" data-listing-reading-time-sort="2" data-listing-word-count-sort="222" data-listing-title-sort="Instruction Tuning" data-listing-filename-sort="inst_tune.qmd">
|
||||
<td>
|
||||
<a href="../../docs/dataset-formats/inst_tune.html" class="title listing-title">Instruction Tuning</a>
|
||||
</td>
|
||||
<td>
|
||||
<span class="listing-description">Instruction tuning formats for supervised fine-tuning.</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr data-index="2" data-listing-file-modified-sort="1712171163568" data-listing-reading-time-sort="1" data-listing-word-count-sort="47" data-listing-title-sort="Pre-training" data-listing-filename-sort="pretraining.qmd">
|
||||
<tr data-index="0" data-listing-file-modified-sort="1712273633798" data-listing-reading-time-sort="1" data-listing-word-count-sort="47" data-listing-title-sort="Pre-training" data-listing-filename-sort="pretraining.qmd">
|
||||
<td>
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="title listing-title">Pre-training</a>
|
||||
</td>
|
||||
@@ -375,7 +359,23 @@ Description
|
||||
<span class="listing-description">Data format for a pre-training completion task.</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr data-index="3" data-listing-file-modified-sort="1712171163568" data-listing-reading-time-sort="1" data-listing-word-count-sort="3" data-listing-title-sort="Template-Free" data-listing-filename-sort="template_free.qmd">
|
||||
<tr data-index="1" data-listing-file-modified-sort="1712273633798" data-listing-reading-time-sort="2" data-listing-word-count-sort="308" data-listing-title-sort="Instruction Tuning" data-listing-filename-sort="inst_tune.qmd">
|
||||
<td>
|
||||
<a href="../../docs/dataset-formats/inst_tune.html" class="title listing-title">Instruction Tuning</a>
|
||||
</td>
|
||||
<td>
|
||||
<span class="listing-description">Instruction tuning formats for supervised fine-tuning.</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr data-index="2" data-listing-file-modified-sort="1712273633798" data-listing-reading-time-sort="2" data-listing-word-count-sort="235" data-listing-title-sort="Conversation" data-listing-filename-sort="conversation.qmd">
|
||||
<td>
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="title listing-title">Conversation</a>
|
||||
</td>
|
||||
<td>
|
||||
<span class="listing-description">Conversation format for supervised fine-tuning.</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr data-index="3" data-listing-file-modified-sort="1712273633798" data-listing-reading-time-sort="1" data-listing-word-count-sort="3" data-listing-title-sort="Template-Free" data-listing-filename-sort="template_free.qmd">
|
||||
<td>
|
||||
<a href="../../docs/dataset-formats/template_free.html" class="title listing-title">Template-Free</a>
|
||||
</td>
|
||||
@@ -383,7 +383,7 @@ Description
|
||||
<span class="listing-description">Construct prompts without a template.</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr data-index="4" data-listing-file-modified-sort="1712171163568" data-listing-reading-time-sort="1" data-listing-word-count-sort="22" data-listing-title-sort="Custom Pre-Tokenized Dataset" data-listing-filename-sort="tokenized.qmd">
|
||||
<tr data-index="4" data-listing-file-modified-sort="1712273633798" data-listing-reading-time-sort="1" data-listing-word-count-sort="22" data-listing-title-sort="Custom Pre-Tokenized Dataset" data-listing-filename-sort="tokenized.qmd">
|
||||
<td>
|
||||
<a href="../../docs/dataset-formats/tokenized.html" class="title listing-title">Custom Pre-Tokenized Dataset</a>
|
||||
</td>
|
||||
|
||||
@@ -218,8 +218,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -230,8 +230,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -301,6 +301,7 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
<li><a href="#creative_acr.load_critique" id="toc-creative_acr.load_critique" class="nav-link" data-scroll-target="#creative_acr.load_critique">creative_acr.load_critique</a></li>
|
||||
<li><a href="#creative_acr.load_revise" id="toc-creative_acr.load_revise" class="nav-link" data-scroll-target="#creative_acr.load_revise">creative_acr.load_revise</a></li>
|
||||
<li><a href="#metharme" id="toc-metharme" class="nav-link" data-scroll-target="#metharme">metharme</a></li>
|
||||
<li><a href="#how-to-add-custom-prompt-format" id="toc-how-to-add-custom-prompt-format" class="nav-link" data-scroll-target="#how-to-add-custom-prompt-format">How to add custom prompt format</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
@@ -530,6 +531,32 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</div>
|
||||
<div class="sourceCode" id="cb20" data-filename="data.jsonl"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb20-1"><a href="#cb20-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span><span class="dt">"prompt"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"generation"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">}</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="how-to-add-custom-prompt-format" class="level2">
|
||||
<h2 class="anchored" data-anchor-id="how-to-add-custom-prompt-format">How to add custom prompt format</h2>
|
||||
<p>For a dataset that is preprocessed for instruction purposes:</p>
|
||||
<div class="code-with-filename">
|
||||
<div class="code-with-filename-file">
|
||||
<pre><strong>data.jsonl</strong></pre>
|
||||
</div>
|
||||
<div class="sourceCode" id="cb21" data-filename="data.jsonl"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb21-1"><a href="#cb21-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span><span class="dt">"input"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">,</span> <span class="dt">"output"</span><span class="fu">:</span> <span class="st">"..."</span><span class="fu">}</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
</div>
|
||||
<p>You can use this example in your YAML config:</p>
|
||||
<div class="code-with-filename">
|
||||
<div class="code-with-filename-file">
|
||||
<pre><strong>config.yaml</strong></pre>
|
||||
</div>
|
||||
<div class="sourceCode" id="cb22" data-filename="config.yaml"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb22-1"><a href="#cb22-1" aria-hidden="true" tabindex="-1"></a><span class="fu">datasets</span><span class="kw">:</span></span>
|
||||
<span id="cb22-2"><a href="#cb22-2" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="kw">-</span><span class="at"> </span><span class="fu">path</span><span class="kw">:</span><span class="at"> repo</span></span>
|
||||
<span id="cb22-3"><a href="#cb22-3" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">type</span><span class="kw">:</span></span>
|
||||
<span id="cb22-4"><a href="#cb22-4" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">system_prompt</span><span class="kw">:</span><span class="at"> </span><span class="st">""</span></span>
|
||||
<span id="cb22-5"><a href="#cb22-5" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">field_system</span><span class="kw">:</span><span class="at"> system</span></span>
|
||||
<span id="cb22-6"><a href="#cb22-6" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">field_instruction</span><span class="kw">:</span><span class="at"> input</span></span>
|
||||
<span id="cb22-7"><a href="#cb22-7" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">field_output</span><span class="kw">:</span><span class="at"> output</span></span>
|
||||
<span id="cb22-8"><a href="#cb22-8" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">format</span><span class="kw">:</span><span class="at"> </span><span class="st">"[INST] {instruction} [/INST]"</span></span>
|
||||
<span id="cb22-9"><a href="#cb22-9" aria-hidden="true" tabindex="-1"></a><span class="at"> </span><span class="fu">no_input_format</span><span class="kw">:</span><span class="at"> </span><span class="st">"[INST] {instruction} [/INST]"</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
|
||||
</div>
|
||||
<p>See full config options under <a href="../../docs/config.html">here</a>.</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
@@ -218,8 +218,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link active">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -230,8 +230,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link active">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
|
||||
@@ -184,8 +184,8 @@ ul.task-list li input[type="checkbox"] {
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -196,8 +196,8 @@ ul.task-list li input[type="checkbox"] {
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
|
||||
@@ -218,8 +218,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -230,8 +230,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
|
||||
@@ -218,8 +218,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -230,8 +230,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
|
||||
@@ -184,8 +184,8 @@ ul.task-list li input[type="checkbox"] {
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -196,8 +196,8 @@ ul.task-list li input[type="checkbox"] {
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
|
||||
@@ -184,8 +184,8 @@ ul.task-list li input[type="checkbox"] {
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -196,8 +196,8 @@ ul.task-list li input[type="checkbox"] {
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
|
||||
@@ -218,8 +218,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -230,8 +230,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -352,7 +352,7 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</section>
|
||||
<section id="you-may-not-want-prompt-templates" class="level3">
|
||||
<h3 class="anchored" data-anchor-id="you-may-not-want-prompt-templates">You may not want prompt templates</h3>
|
||||
<p>However, there are many situations where you don’t want to use one of these formats or templates (I usually don’t!). This is because they can:</p>
|
||||
<p>However, there are many situations where you don’t want to use one of these formats or templates. This is because they can:</p>
|
||||
<ul>
|
||||
<li>Add unnecessary boilerplate to your prompts.</li>
|
||||
<li>Create artifacts like special delimiters <code><|im_start|></code> that can quickly become footguns if you don’t include them correctly at inference time.</li>
|
||||
|
||||
@@ -184,8 +184,8 @@ ul.task-list li input[type="checkbox"] {
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -196,8 +196,8 @@ ul.task-list li input[type="checkbox"] {
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
|
||||
@@ -218,8 +218,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -230,8 +230,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
|
||||
@@ -184,8 +184,8 @@ ul.task-list li input[type="checkbox"] {
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -196,8 +196,8 @@ ul.task-list li input[type="checkbox"] {
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
|
||||
@@ -184,8 +184,8 @@ ul.task-list li input[type="checkbox"] {
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -196,8 +196,8 @@ ul.task-list li input[type="checkbox"] {
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
|
||||
@@ -218,8 +218,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
@@ -230,8 +230,8 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
<div class="sidebar-item-container">
|
||||
<a href="../docs/dataset-formats/pretraining.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Pre-training</span></a>
|
||||
<a href="../docs/dataset-formats/conversation.html" class="sidebar-item-text sidebar-link">
|
||||
<span class="menu-text">Conversation</span></a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="sidebar-item">
|
||||
|
||||
Reference in New Issue
Block a user