more autodoc progress

_quarto.yml

@@ -14,10 +14,17 @@ quartodoc:
         - train
         - evaluate
         - datasets
+        - convert
+        - prompt_tokenizers
+        - logging_config
         - core.trainer_builder
         - core.training_args
         - core.chat.messages
+        - core.chat.format.chatml
+        - core.chat.format.llama3x
+        - core.chat.format.shared
         - core.datasets.chat
+        - core.datasets.transforms.chat_builder
     - title: CLI
       desc: Command-line interface
       contents:
@@ -49,16 +56,27 @@ quartodoc:
         - prompt_strategies.chat_template
         - prompt_strategies.alpaca_chat
         - prompt_strategies.alpaca_instruct
+        - prompt_strategies.alpaca_w_system
         - prompt_strategies.user_defined
         - prompt_strategies.llama2_chat
         - prompt_strategies.completion
         - prompt_strategies.input_output
         - prompt_strategies.stepwise_supervised
+        - prompt_strategies.metharme
+        - prompt_strategies.orcamini
+        - prompt_strategies.pygmalion
         - prompt_strategies.messages.chat
         - prompt_strategies.dpo.chat_template
         - prompt_strategies.dpo.llama3
+        - prompt_strategies.dpo.chatml
+        - prompt_strategies.dpo.zephyr
+        - prompt_strategies.dpo.user_defined
+        - prompt_strategies.dpo.passthrough
         - prompt_strategies.kto.llama3
+        - prompt_strategies.kto.chatml
+        - prompt_strategies.kto.user_defined
         - prompt_strategies.orpo.chat_template
+        - prompt_strategies.bradley_terry.llama3
     - title: Kernels
       desc: Low-level performance optimizations
       contents:
@@ -86,6 +104,7 @@ quartodoc:
         - monkeypatch.unsloth_
         - monkeypatch.attention.mllama
         - monkeypatch.data.batch_dataset_fetcher
+        - monkeypatch.mixtral
     - title: Utils
       desc: Utility functions
       contents:
@@ -93,6 +112,9 @@ quartodoc:
         - utils.tokenization
         - utils.chat_templates
         - utils.lora
+        - utils.lora_embeddings
+        - utils.model_shard_quant
+        - utils.bench
         - utils.freeze
         - utils.trainer
         - utils.schedulers
@@ -101,6 +123,7 @@ quartodoc:
         - utils.optimizers.adopt
         - utils.data.pretraining
         - utils.data.sft
+        - utils.gradient_checkpointing.unsloth
     - title: Integrations
       desc: Third-party integrations and extensions
       contents:
@@ -111,6 +134,12 @@ quartodoc:
         - integrations.liger.args
         - integrations.lm_eval.args
         - integrations.spectrum.args
+    - title: Common
+      desc: Common utilities and shared functionality
+      contents:
+        - common.architectures
+        - common.const
+        - common.datasets
     - title: Models
       desc: Custom model implementations
       contents:

docs/api/README.md

@@ -1,59 +0,0 @@
-# Axolotl API Documentation with quartodoc
-
-This directory contains the API documentation for Axolotl, automatically generated using quartodoc.
-
-## Setup
-
-1. Make sure quartodoc is installed:
-   ```
-   pip install quartodoc
-   ```
-
-2. Install Quarto (required to render the documentation):
-   ```
-   # Download and install the latest Quarto release
-   # Visit https://quarto.org/docs/get-started/ for installation instructions
-   ```
-
-## Generating Documentation
-
-Run the documentation generation script:
-```
-python scripts/generate_docs.py
-```
-
-This will:
-- Read the configuration from `_quarto.yml`
-- Extract documentation from the Python source code
-- Generate Quarto markdown files (.qmd) in the `docs/api` directory
-
-## Preview the Documentation
-
-After generating the documentation, preview it with:
-```
-quarto preview
-```
-
-## Building the Site
-
-Build the complete site with:
-```
-quarto render
-```
-
-This will create a `_site` directory with the static HTML site.
-
-## Configuration
-
-The documentation generation is configured in two places:
-
-1. `_quarto.yml` - Contains the `quartodoc` section that defines which modules to document
-2. The API section in the Quarto website sidebar configuration (also in `_quarto.yml`)
-
-## Customization
-
-To customize the documentation, you can:
-
-1. Add more modules to document in the `quartodoc` section of `_quarto.yml`
-2. Create template files in the `quartodoc_templates` directory
-3. Adjust the layout in the Quarto configuration

quartodoc.yml

@@ -1,47 +0,0 @@
-project:
-  title: Axolotl API Reference
-  description: API documentation for Axolotl
-
-repo:
-  url: https://github.com/axolotl-ai-cloud/axolotl
-
-packages:
-  - package: axolotl
-    source_path: src/axolotl
-    url_path: /api
-
-    sections:
-      - title: Core
-        source_path: core
-        url_path: core
-
-      - title: CLI
-        source_path: cli
-        url_path: cli
-
-      - title: Prompt Strategies
-        source_path: prompt_strategies
-        url_path: prompt_strategies
-
-      - title: Utils
-        source_path: utils
-        url_path: utils
-
-      - title: Integrations
-        source_path: integrations
-        url_path: integrations
-
-    contents:
-      - "__init__.py"
-      - "train.py"
-      - "evaluate.py"
-      - "datasets.py"
-      - "prompters.py"
-
-renderer:
-  output_dir: docs/api
-  schema_dir: docs/schema
-  number_section_groups: true
-  template_paths:
-    - quartodoc_templates
-  quiet: false