fix: add missing shards_idx, preprocess_shards to docs and validator (#2331)

This commit is contained in:
NanoCode012
2025-02-14 05:28:21 +07:00
committed by GitHub
parent aa45fed451
commit 2e57391bf8
2 changed files with 7 additions and 1 deletions

View File

@@ -169,6 +169,7 @@ class SFTDataset(BaseModel):
type: Optional[Union[str, UserDefinedPrompterType]] = None
input_transform: Optional[str] = None
shards: Optional[int] = None
shards_idx: Optional[int] = None
preprocess_shards: Optional[int] = None
conversation: Optional[str] = None
# Do not make this too strict or it will break the validator to choose different dataset class