support for batched sharegpt tokenization to skip bad data

2023-10-06 15:03:07 -04:00
parent 669f1d052c
commit b4d84d56d5
7 changed files with 170 additions and 96 deletions
--- a/.pylintrc
+++ b/.pylintrc
@@ -12,3 +12,4 @@ generated-members=numpy.*, torch.*
 disable=missing-function-docstring, line-too-long, import-error,
    too-many-arguments, too-many-locals, too-many-statements, too-many-branches, too-few-public-methods,
    too-many-instance-attributes, fixme, import-outside-toplevel, logging-fstring-interpolation,
+    too-many-nested-blocks,