From ba9a867adb39dd5951be45707613a80d542678e1 Mon Sep 17 00:00:00 2001 From: Dan Saunders Date: Mon, 17 Mar 2025 12:26:58 -0400 Subject: [PATCH] more autodoc progress --- _quarto.yml | 29 +++++++++++++++++++++++ docs/api/README.md | 59 ---------------------------------------------- quartodoc.yml | 47 ------------------------------------ 3 files changed, 29 insertions(+), 106 deletions(-) delete mode 100644 docs/api/README.md delete mode 100644 quartodoc.yml diff --git a/_quarto.yml b/_quarto.yml index a6d2f7dc0..30a87080e 100644 --- a/_quarto.yml +++ b/_quarto.yml @@ -14,10 +14,17 @@ quartodoc: - train - evaluate - datasets + - convert + - prompt_tokenizers + - logging_config - core.trainer_builder - core.training_args - core.chat.messages + - core.chat.format.chatml + - core.chat.format.llama3x + - core.chat.format.shared - core.datasets.chat + - core.datasets.transforms.chat_builder - title: CLI desc: Command-line interface contents: @@ -49,16 +56,27 @@ quartodoc: - prompt_strategies.chat_template - prompt_strategies.alpaca_chat - prompt_strategies.alpaca_instruct + - prompt_strategies.alpaca_w_system - prompt_strategies.user_defined - prompt_strategies.llama2_chat - prompt_strategies.completion - prompt_strategies.input_output - prompt_strategies.stepwise_supervised + - prompt_strategies.metharme + - prompt_strategies.orcamini + - prompt_strategies.pygmalion - prompt_strategies.messages.chat - prompt_strategies.dpo.chat_template - prompt_strategies.dpo.llama3 + - prompt_strategies.dpo.chatml + - prompt_strategies.dpo.zephyr + - prompt_strategies.dpo.user_defined + - prompt_strategies.dpo.passthrough - prompt_strategies.kto.llama3 + - prompt_strategies.kto.chatml + - prompt_strategies.kto.user_defined - prompt_strategies.orpo.chat_template + - prompt_strategies.bradley_terry.llama3 - title: Kernels desc: Low-level performance optimizations contents: @@ -86,6 +104,7 @@ quartodoc: - monkeypatch.unsloth_ - monkeypatch.attention.mllama - monkeypatch.data.batch_dataset_fetcher + - monkeypatch.mixtral - title: Utils desc: Utility functions contents: @@ -93,6 +112,9 @@ quartodoc: - utils.tokenization - utils.chat_templates - utils.lora + - utils.lora_embeddings + - utils.model_shard_quant + - utils.bench - utils.freeze - utils.trainer - utils.schedulers @@ -101,6 +123,7 @@ quartodoc: - utils.optimizers.adopt - utils.data.pretraining - utils.data.sft + - utils.gradient_checkpointing.unsloth - title: Integrations desc: Third-party integrations and extensions contents: @@ -111,6 +134,12 @@ quartodoc: - integrations.liger.args - integrations.lm_eval.args - integrations.spectrum.args + - title: Common + desc: Common utilities and shared functionality + contents: + - common.architectures + - common.const + - common.datasets - title: Models desc: Custom model implementations contents: diff --git a/docs/api/README.md b/docs/api/README.md deleted file mode 100644 index f8a931e22..000000000 --- a/docs/api/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Axolotl API Documentation with quartodoc - -This directory contains the API documentation for Axolotl, automatically generated using quartodoc. - -## Setup - -1. Make sure quartodoc is installed: - ``` - pip install quartodoc - ``` - -2. Install Quarto (required to render the documentation): - ``` - # Download and install the latest Quarto release - # Visit https://quarto.org/docs/get-started/ for installation instructions - ``` - -## Generating Documentation - -Run the documentation generation script: -``` -python scripts/generate_docs.py -``` - -This will: -- Read the configuration from `_quarto.yml` -- Extract documentation from the Python source code -- Generate Quarto markdown files (.qmd) in the `docs/api` directory - -## Preview the Documentation - -After generating the documentation, preview it with: -``` -quarto preview -``` - -## Building the Site - -Build the complete site with: -``` -quarto render -``` - -This will create a `_site` directory with the static HTML site. - -## Configuration - -The documentation generation is configured in two places: - -1. `_quarto.yml` - Contains the `quartodoc` section that defines which modules to document -2. The API section in the Quarto website sidebar configuration (also in `_quarto.yml`) - -## Customization - -To customize the documentation, you can: - -1. Add more modules to document in the `quartodoc` section of `_quarto.yml` -2. Create template files in the `quartodoc_templates` directory -3. Adjust the layout in the Quarto configuration diff --git a/quartodoc.yml b/quartodoc.yml deleted file mode 100644 index fbbb93852..000000000 --- a/quartodoc.yml +++ /dev/null @@ -1,47 +0,0 @@ -project: - title: Axolotl API Reference - description: API documentation for Axolotl - -repo: - url: https://github.com/axolotl-ai-cloud/axolotl - -packages: - - package: axolotl - source_path: src/axolotl - url_path: /api - - sections: - - title: Core - source_path: core - url_path: core - - - title: CLI - source_path: cli - url_path: cli - - - title: Prompt Strategies - source_path: prompt_strategies - url_path: prompt_strategies - - - title: Utils - source_path: utils - url_path: utils - - - title: Integrations - source_path: integrations - url_path: integrations - - contents: - - "__init__.py" - - "train.py" - - "evaluate.py" - - "datasets.py" - - "prompters.py" - -renderer: - output_dir: docs/api - schema_dir: docs/schema - number_section_groups: true - template_paths: - - quartodoc_templates - quiet: false