Create preprocess CLI (#785)

* Create preprocess CLI * Print prompt template if debugging * Add print for unsupported prompters * Formatting * Formatting * Refactor variables * Formatting * Formatting * Formatting * Formatting
2023-10-26 15:35:42 +02:00
parent 05bd6f1122
commit e50ab072e2
9 changed files with 354 additions and 190 deletions
--- a/scripts/finetune.py
+++ b/scripts/finetune.py
@@ -45,8 +45,6 @@ def do_cli(config: Path = Path("examples/"), **kwargs):
        shard(cfg=parsed_cfg, cli_args=parsed_cli_args)
    else:
        dataset_meta = load_datasets(cfg=parsed_cfg, cli_args=parsed_cli_args)
-        if parsed_cli_args.prepare_ds_only:
-            return
        train(cfg=parsed_cfg, cli_args=parsed_cli_args, dataset_meta=dataset_meta)