add two checks to handle legacy format interleaved multimodal ds (#2721) [skip ci]
* add two checks to handle legacy format interleaved ds * fix: add warning about multiple image using legacy format --------- Co-authored-by: NanoCode012 <nano@axolotl.ai>
This commit is contained in:
@@ -1,5 +1,6 @@
|
|||||||
"""Module containing ProcessingStrategy classes and its derivative for different MultiModal Model types"""
|
"""Module containing ProcessingStrategy classes and its derivative for different MultiModal Model types"""
|
||||||
|
|
||||||
|
import logging
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
@@ -9,6 +10,8 @@ from torch import Tensor
|
|||||||
from transformers import ProcessorMixin
|
from transformers import ProcessorMixin
|
||||||
from transformers.image_utils import load_image
|
from transformers.image_utils import load_image
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class ProcessingStrategy:
|
class ProcessingStrategy:
|
||||||
"""Base Processing Strategy class"""
|
"""Base Processing Strategy class"""
|
||||||
@@ -112,7 +115,9 @@ class ProcessingStrategy:
|
|||||||
)
|
)
|
||||||
|
|
||||||
processed_example = None
|
processed_example = None
|
||||||
if "messages" in example: # OpenAI format
|
if (
|
||||||
|
"messages" in example and example["messages"] is not None
|
||||||
|
): # OpenAI format
|
||||||
processed_example = example
|
processed_example = example
|
||||||
else: # Legacy format
|
else: # Legacy format
|
||||||
processed_example = convert_legacy_format(example)
|
processed_example = convert_legacy_format(example)
|
||||||
@@ -132,10 +137,17 @@ class ProcessingStrategy:
|
|||||||
break
|
break
|
||||||
|
|
||||||
# if the image key exists, add the image to the first message
|
# if the image key exists, add the image to the first message
|
||||||
if image_key is not None:
|
if image_key is not None and processed_example[image_key] is not None:
|
||||||
# TODO: check if it's normal to be single image only for common datasets
|
# TODO: check if it's normal to be single image only for common datasets
|
||||||
# From observation, it's usually a list of single image but some datasets may have several columns for images
|
# From observation, it's usually a list of single image but some datasets may have several columns for images
|
||||||
# Temporary solution: take the first image and suggest people convert their datasets to use multi-content Messages
|
# Temporary solution: take the first image and suggest people convert their datasets to use multi-content Messages
|
||||||
|
if len(processed_example[image_key]) > 0:
|
||||||
|
LOG.warning(
|
||||||
|
f"Found {len(processed_example[image_key])} images in a sample. Using the first one."
|
||||||
|
"If you are using a dataset with multiple images per sample, please convert it to use multi-content Messages."
|
||||||
|
"See https://docs.axolotl.ai/docs/multimodal.html#dataset-format"
|
||||||
|
)
|
||||||
|
|
||||||
image_value = processed_example[image_key][0]
|
image_value = processed_example[image_key][0]
|
||||||
|
|
||||||
# Handle image loading (Image, url, path, base64)
|
# Handle image loading (Image, url, path, base64)
|
||||||
|
|||||||
Reference in New Issue
Block a user