From eb72c6a1792eaf9237cc620f6899b0e35405e377 Mon Sep 17 00:00:00 2001
From: ChenX17 <xichen7@link.cuhk.edu.cn>
Date: Sat, 13 Jul 2024 10:15:44 +0000
Subject: [PATCH 1/2] [Feature] Add LLaST

---
 xtuner/model/llast.py | 294 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 294 insertions(+)
 create mode 100644 xtuner/model/llast.py

diff --git a/xtuner/model/llast.py b/xtuner/model/llast.py
new file mode 100644
index 000000000..246fd7cde
--- /dev/null
+++ b/xtuner/model/llast.py
@@ -0,0 +1,294 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from collections import OrderedDict
+
+import torch
+import torch.nn as nn
+from mmengine.config import Config, ConfigDict
+from mmengine.model import BaseModel
+from peft import get_peft_model, prepare_model_for_kbit_training
+from transformers import PretrainedConfig, PreTrainedModel
+from transformers.activations import ACT2FN
+
+from xtuner.dataset.llast import prepare_inputs_labels_for_llast
+from xtuner.registry import BUILDER
+from .modules import dispatch_modules
+from .utils import (LoadWoInit, find_all_linear_names,
+                    get_peft_model_state_dict, guess_load_checkpoint,
+                    make_inputs_require_grad, traverse_dict)
+
+
+class AudioProjectorConfig(PretrainedConfig):
+    model_type = 'projector'
+    _auto_class = 'AutoConfig'
+
+    def __init__(
+        self,
+        audio_hidden_size=4096,
+        llm_hidden_size=4096,
+        depth=2,
+        hidden_act='gelu',
+        bias=True,
+        **kwargs,
+    ):
+        self.audio_hidden_size = audio_hidden_size
+        self.llm_hidden_size = llm_hidden_size
+        self.depth = depth
+        self.hidden_act = hidden_act
+        self.bias = bias
+        super().__init__(**kwargs)
+
+
+class AudioEncoder(PreTrainedModel):
+    _auto_class = 'AutoModel'
+    config_class = AudioProjectorConfig
+    base_model_prefix = 'model'
+    supports_gradient_checkpointing = True
+
+    def __init__(self, config: AudioProjectorConfig) -> None:
+        super().__init__(config)
+        self.gradient_checkpointing = False
+        print('*' * 30)
+        print(config.audio_hidden_size, config.llm_hidden_size)
+        modules = [nn.Linear(config.audio_hidden_size, config.llm_hidden_size)]
+        for _ in range(1, config.depth):
+            modules.append(ACT2FN[config.hidden_act])
+            modules.append(
+                nn.Linear(
+                    config.llm_hidden_size,
+                    config.llm_hidden_size,
+                    bias=config.bias))
+        self.model = nn.Sequential(*modules)
+
+    def enable_input_require_grads(self):
+
+        def make_inputs_require_grad(module, input, output):
+            output.requires_grad_(True)
+
+        self.model.register_forward_hook(make_inputs_require_grad)
+
+    def _set_gradient_checkpointing(self, module, value=False):
+        if isinstance(module, AudioProjectorConfig):
+            module.gradient_checkpointing = value
+
+    def forward(self, x):
+        if self.gradient_checkpointing and self.training:
+            layer_outputs = torch.utils.checkpoint.checkpoint(self.model, x)
+        else:
+            layer_outputs = self.model(x)
+        return layer_outputs
+
+
+class LLaSTModel(BaseModel):
+    """Implementation of LLaST.
+
+    Acknowledge: LLaVA: Visual Instruction Tuning
+    (https://llava-vl.github.io/)
+    """
+
+    def __init__(
+        self,
+        llm,
+        speech_encoder,
+        freeze_llm=False,
+        freeze_speech_encoder=False,
+        speech_select_layer=-1,
+        pretrained_pth=None,
+        projector_depth=2,
+        llm_lora=None,
+        speech_encoder_lora=None,
+        use_activation_checkpointing=True,
+    ):
+        super().__init__()
+        self.freeze_llm = freeze_llm
+        self.freeze_speech_encoder = freeze_speech_encoder
+        with LoadWoInit():
+            self.llm = self._build_from_cfg_or_module(llm)
+            self.speech_encoder = self._build_from_cfg_or_module(
+                speech_encoder)
+
+        self.llm.config.use_cache = False
+        dispatch_modules(self.llm)
+
+        projector_config = AudioProjectorConfig(
+            audio_hidden_size=self.speech_encoder.config.hidden_size,
+            llm_hidden_size=self.llm.config.hidden_size,
+            depth=projector_depth)
+        self.projector = AudioEncoder(projector_config).to(
+            self.speech_encoder.dtype)
+
+        if self.freeze_llm:
+            self.llm.requires_grad_(False)
+        if self.freeze_speech_encoder:
+            self.speech_encoder.requires_grad_(False)
+
+        if use_activation_checkpointing:
+            # For backward compatibility
+            if hasattr(self.llm, 'enable_input_require_grads'):
+                self.llm.enable_input_require_grads()
+            else:
+                self.llm.get_input_embeddings().register_forward_hook(
+                    make_inputs_require_grad)
+            if hasattr(self.speech_encoder, 'enable_input_require_grads'):
+                self.speech_encoder.enable_input_require_grads()
+            else:
+                self.speech_encoder.get_input_embeddings(
+                ).register_forward_hook(make_inputs_require_grad)
+            self.projector.enable_input_require_grads()
+
+            # enable gradient (activation) checkpointing for memory efficiency
+            self.gradient_checkpointing_enable()
+
+        self.use_llm_lora = llm_lora is not None
+        self.use_speech_encoder_lora = speech_encoder_lora is not None
+
+        if self.use_llm_lora:
+            self._prepare_llm_for_lora(llm_lora, use_activation_checkpointing)
+        if self.use_speech_encoder_lora:
+            self._prepare_speech_encoder_for_lora(
+                speech_encoder_lora, use_activation_checkpointing)
+
+        if pretrained_pth is not None:
+            pretrained_state_dict = guess_load_checkpoint(pretrained_pth)
+
+            out_str = self.load_state_dict(pretrained_state_dict, strict=False)
+            assert len(out_str.unexpected_keys) == 0, out_str.unexpected_keys
+            print(f'Load pretrained weight from {pretrained_pth}')
+
+        self.speech_select_layer = speech_select_layer
+
+        self._is_init = True
+
+    def _parse_lora_config(self, lora_config):
+        if isinstance(lora_config, dict) or isinstance(
+                lora_config, Config) or isinstance(lora_config, ConfigDict):
+            lora_config = BUILDER.build(lora_config)
+        return lora_config
+
+    def gradient_checkpointing_enable(self):
+        self.activation_checkpointing_enable()
+
+    def activation_checkpointing_enable(self):
+        self.llm.gradient_checkpointing_enable(
+            gradient_checkpointing_kwargs={'use_reentrant': False})
+        self.speech_encoder.gradient_checkpointing_enable(
+            gradient_checkpointing_kwargs={'use_reentrant': False})
+        self.projector.gradient_checkpointing_enable(
+            gradient_checkpointing_kwargs={'use_reentrant': False})
+
+    def gradient_checkpointing_disable(self):
+        self.activation_checkpointing_disable()
+
+    def activation_checkpointing_disable(self):
+        self.llm.gradient_checkpointing_disable()
+        self.speech_encoder.gradient_checkpointing_disable()
+        self.projector.gradient_checkpointing_disable()
+
+    def init_weights(self):
+        pass
+
+    def state_dict(self, *args, **kwargs):
+        state_dict = super().state_dict(*args, **kwargs)
+        to_return = OrderedDict()
+        # Step 1. speech_encoder
+        if self.use_speech_encoder_lora:
+            to_return.update(
+                get_peft_model_state_dict(
+                    self.speech_encoder, state_dict=state_dict))
+        elif not self.freeze_speech_encoder:
+            to_return.update({
+                k: v
+                for k, v in state_dict.items() if 'speech_encoder.' in k
+            })
+        # Step 2. LLM
+        if self.use_llm_lora:
+            to_return.update(
+                get_peft_model_state_dict(self.llm, state_dict=state_dict))
+        elif not self.freeze_llm:
+            to_return.update(
+                {k: v
+                 for k, v in state_dict.items() if 'llm.' in k})
+        # Step 3. Projector
+        to_return.update(
+            {k: v
+             for k, v in state_dict.items() if 'projector.' in k})
+        return to_return
+
+    def _build_from_cfg_or_module(self, cfg_or_mod):
+        if isinstance(cfg_or_mod, nn.Module):
+            return cfg_or_mod
+        elif isinstance(cfg_or_mod, dict):
+            traverse_dict(cfg_or_mod)
+            return BUILDER.build(cfg_or_mod)
+        else:
+            raise NotImplementedError
+
+    def _prepare_llm_for_lora(self,
+                              lora_config,
+                              use_activation_checkpointing=True):
+        lora_config = self._parse_lora_config(lora_config)
+        self.llm = prepare_model_for_kbit_training(
+            self.llm, use_activation_checkpointing)
+        if lora_config.target_modules is None:
+            modules = find_all_linear_names(self.llm)
+            lora_config.target_modules = modules
+        self.llm = get_peft_model(self.llm, lora_config)
+
+    def _prepare_speech_encoder_for_lora(self,
+                                         lora_config,
+                                         use_activation_checkpointing=True):
+        lora_config = self._parse_lora_config(lora_config)
+        if lora_config.target_modules is None:
+            modules = find_all_linear_names(self.speech_encoder)
+            lora_config.target_modules = modules
+        self.speech_encoder = get_peft_model(self.speech_encoder, lora_config)
+
+    def forward(self, data, data_samples=None, mode='loss'):
+        if 'audio_tokens' in data:
+            data['audio_tokens'] = data['audio_tokens'].to(
+                self.speech_encoder.encoder.conv1.weight.dtype)
+            batch_size = data['audio_tokens'].shape[0]
+            decoder_input_ids = torch.tensor([
+                [1] * batch_size
+            ]) * self.speech_encoder.config.decoder_start_token_id
+
+            audio_outputs = self.speech_encoder(
+                data['audio_tokens'],
+                decoder_input_ids=decoder_input_ids.to(
+                    data['audio_tokens'].device),
+                output_hidden_states=True).encoder_last_hidden_state
+
+            audio_outputs = audio_outputs[:, :max(data['audio_lens']), :]
+            audio_tokens = self.projector(audio_outputs)
+            data['audio_tokens'] = audio_tokens
+            data = prepare_inputs_labels_for_llast(llm=self.llm, **data)
+
+        if mode == 'loss':
+            return self.compute_loss(data, data_samples)
+        elif mode == 'predict':
+            return self.predict(data, data_samples)
+        elif mode == 'tensor':
+            return self._forward(data, data_samples)
+        else:
+            raise NotImplementedError
+
+    def _forward(self, data, data_samples=None):
+
+        outputs = self.llm(**data)
+
+        return outputs
+
+    def predict(self, data, data_samples=None):
+        outputs = self.llm(**data)
+        logits_dict = [{'logits': logits} for logits in outputs.logits]
+        return logits_dict
+
+    def compute_loss(self, data, data_samples=None):
+        outputs = self.llm(**data)
+        loss_dict = {'loss': outputs.loss}
+        return loss_dict
+
+    def __getattr__(self, name: str):
+        try:
+            return super().__getattr__(name)
+        except AttributeError:
+            return getattr(self.llm, name)

From 0958b5d6dfd0982b5aaca855bcee3f721641bd95 Mon Sep 17 00:00:00 2001
From: ChenX17 <xichen7@link.cuhk.edu.cn>
Date: Sat, 13 Jul 2024 11:51:27 +0000
Subject: [PATCH 2/2] Update LLaST implementation

---
 .pre-commit-config.yaml                       |   2 +-
 xtuner/dataset/__init__.py                    |   3 +-
 xtuner/dataset/collate_fns/__init__.py        |   5 +-
 .../dataset/collate_fns/llast_collate_fn.py   |  60 +++
 xtuner/dataset/huggingface.py                 |  19 +-
 xtuner/dataset/llast.py                       | 492 ++++++++++++++++++
 xtuner/dataset/utils.py                       |  20 +-
 xtuner/engine/__init__.py                     |   5 +-
 xtuner/engine/runner/__init__.py              |   3 +-
 xtuner/engine/runner/llast_loops.py           | 166 ++++++
 xtuner/evaluation/metrics/__init__.py         |   3 +-
 xtuner/evaluation/metrics/sacrebleu.py        | 182 +++++++
 xtuner/evaluation/metrics/sacrebleu_metric.py | 112 ++++
 xtuner/model/__init__.py                      |   3 +-
 xtuner/model/llast.py                         |   1 +
 xtuner/tools/process_llast_data.py            |  33 ++
 xtuner/utils/__init__.py                      |   9 +-
 xtuner/utils/constants.py                     |   3 +
 xtuner/utils/templates.py                     |   4 +
 19 files changed, 1108 insertions(+), 17 deletions(-)
 create mode 100644 xtuner/dataset/collate_fns/llast_collate_fn.py
 create mode 100644 xtuner/dataset/llast.py
 create mode 100644 xtuner/engine/runner/llast_loops.py
 create mode 100644 xtuner/evaluation/metrics/sacrebleu.py
 create mode 100644 xtuner/evaluation/metrics/sacrebleu_metric.py
 create mode 100644 xtuner/tools/process_llast_data.py

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f6bbfd633..98a23e46e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,7 +4,7 @@ repos:
     rev: 5.0.4
     hooks:
       - id: flake8
-        args: ["--exclude=xtuner/model/transformers_models/*"]
+        args: ["--exclude=xtuner/model/transformers_models/*,xtuner/evaluation/metrics/sacrebleu.py"]
   - repo: https://github.com/PyCQA/isort
     rev: 5.11.5
     hooks:
diff --git a/xtuner/dataset/__init__.py b/xtuner/dataset/__init__.py
index 2ad3d7bd9..a8e2590b3 100644
--- a/xtuner/dataset/__init__.py
+++ b/xtuner/dataset/__init__.py
@@ -8,6 +8,7 @@
                           load_intern_repo_untokenized_dataset)
 from .internvl_dataset import InternVL_V1_5_Dataset
 from .json_dataset import load_json_file
+from .llast import LLaSTDataset
 from .llava import LLaVADataset
 from .modelscope import process_ms_dataset
 from .moss_sft import MOSSSFTDataset
@@ -25,5 +26,5 @@
     'load_intern_repo_tokenized_dataset',
     'load_intern_repo_untokenized_dataset', 'build_packed_dataset',
     'RefCOCOJsonDataset', 'RefCOCOJsonEvalDataset', 'InvRefCOCOJsonDataset',
-    'load_json_file', 'InternVL_V1_5_Dataset'
+    'load_json_file', 'InternVL_V1_5_Dataset', 'LLaSTDataset'
 ]
diff --git a/xtuner/dataset/collate_fns/__init__.py b/xtuner/dataset/collate_fns/__init__.py
index 96652b259..86b8e29cf 100644
--- a/xtuner/dataset/collate_fns/__init__.py
+++ b/xtuner/dataset/collate_fns/__init__.py
@@ -1,5 +1,8 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 from .default_collate_fn import default_collate_fn
+from .llast_collate_fn import llast_audiomask_mel_collate_fn
 from .mmlu_collate_fn import mmlu_collate_fn
 
-__all__ = ['default_collate_fn', 'mmlu_collate_fn']
+__all__ = [
+    'default_collate_fn', 'mmlu_collate_fn', 'llast_audiomask_mel_collate_fn'
+]
diff --git a/xtuner/dataset/collate_fns/llast_collate_fn.py b/xtuner/dataset/collate_fns/llast_collate_fn.py
new file mode 100644
index 000000000..01d926b3d
--- /dev/null
+++ b/xtuner/dataset/collate_fns/llast_collate_fn.py
@@ -0,0 +1,60 @@
+# Copyright (c) LLaST. All rights reserved.
+# Copyright (c) OpenMMLab. All rights reserved.
+from typing import Dict, Sequence
+
+import torch
+from torch.nn.utils.rnn import pad_sequence
+
+from xtuner.utils import (DEFAULT_PAD_TOKEN_INDEX, IGNORE_INDEX,
+                          LLAST_AUDIO_PADDING_TOKEN_INDEX)
+
+
+def llast_audiomask_mel_collate_fn(
+        instances: Sequence[Dict],
+        pad_index: int = DEFAULT_PAD_TOKEN_INDEX,
+        return_hf_format: bool = False) -> Dict[str, torch.Tensor]:
+    """Add audio tokens and conduct padding operation."""
+    input_ids = []
+    labels = []
+    feats_lens = []
+    has_audio = any(inst.get('audio_tokens') is not None for inst in instances)
+
+    if has_audio:
+        audio_tokens = []
+    for example in instances:
+        input_ids.append(torch.tensor(example['input_ids']))
+        labels.append(torch.tensor(example['labels']))
+        if has_audio:
+            audio_tokens.append(example['audio_tokens'])
+        feats_lens.append(torch.tensor(example['audio_lens']))
+    if len(instances) > 1:
+        input_ids = pad_sequence(
+            input_ids, batch_first=True, padding_value=pad_index)
+        labels = pad_sequence(
+            labels, batch_first=True, padding_value=IGNORE_INDEX)
+        # padding audio tokens
+        padded_audio_tokens = pad_sequence(
+            audio_tokens,
+            batch_first=True,
+            padding_value=LLAST_AUDIO_PADDING_TOKEN_INDEX)
+
+    else:
+        input_ids = torch.stack(input_ids)
+        labels = torch.stack(labels)
+        padded_audio_tokens = torch.stack(audio_tokens)
+
+    data_dict = {
+        'input_ids': input_ids,
+        'attention_mask': input_ids.ne(pad_index),
+        'labels': labels
+    }
+
+    if has_audio:
+        audio_lens = torch.stack(feats_lens)
+        data_dict['audio_tokens'] = padded_audio_tokens
+        data_dict['audio_lens'] = audio_lens
+
+    if return_hf_format:
+        return data_dict
+    else:
+        return {'data': data_dict, 'data_samples': instances}
diff --git a/xtuner/dataset/huggingface.py b/xtuner/dataset/huggingface.py
index c44e88688..86d04641e 100644
--- a/xtuner/dataset/huggingface.py
+++ b/xtuner/dataset/huggingface.py
@@ -65,8 +65,8 @@ def add_template_to_dataset(dataset, template_map_fn, map_num_proc):
 
 
 def tokenize_dataset(dataset, tokenizer, max_length, with_image_token,
-                     input_ids_with_output, remove_unused_columns,
-                     map_num_proc):
+                     with_audio_token, input_ids_with_output,
+                     remove_unused_columns, map_num_proc):
     assert (tokenizer is not None) and (max_length is not None), \
         f'({tokenizer}, {max_length})'
     if isinstance(tokenizer, dict) or isinstance(
@@ -78,6 +78,7 @@ def tokenize_dataset(dataset, tokenizer, max_length, with_image_token,
             tokenizer=tokenizer,
             max_length=max_length,
             with_image_token=with_image_token,
+            with_audio_token=with_audio_token,
             input_ids_with_output=input_ids_with_output),
         remove_columns=list(dataset.column_names)
         if remove_unused_columns else None,
@@ -112,6 +113,7 @@ def process(dataset,
             use_varlen_attn=False,
             input_ids_with_output=True,
             with_image_token=False,
+            with_audio_token=False,
             map_num_proc=32):
     """Post-process the dataset loaded from the Hugging Face Hub, or a local
     dataset.
@@ -153,6 +155,9 @@ def process(dataset,
         with_image_token: Whether to convert DEFAULT_IMAGE_TOKEN to
             IMAGE_TOKEN_INDEX. Typically set it to True during the training
             of VLM.
+        with_audio_token: Whether to convert DEFAULT_AUDIO_TOKEN to
+            LLAST_AUDIO_TOKEN_INDEX. Typically set it to True during the
+            training of SLM.
         map_num_proc: Max number of processes when mapping the dataset.
     """
     if use_varlen_attn:
@@ -197,7 +202,8 @@ def process(dataset,
 
     if do_dataset_tokenization:
         dataset = tokenize_dataset(dataset, tokenizer, max_length,
-                                   with_image_token, input_ids_with_output,
+                                   with_image_token, with_audio_token,
+                                   input_ids_with_output,
                                    remove_unused_columns, map_num_proc)
 
     if input_ids_with_output:
@@ -213,7 +219,7 @@ def process(dataset,
                                shuffle_before_pack, map_num_proc)
 
     # add 'length'
-    dataset = dataset.map(get_lengths, num_proc=map_num_proc)
+    dataset = dataset.map(get_lengths, num_proc=1)
     setattr(dataset, 'length', dataset['length'])
 
     return dataset
@@ -234,6 +240,7 @@ def process_hf_dataset(dataset,
                        use_varlen_attn=False,
                        input_ids_with_output=True,
                        with_image_token=False,
+                       with_audio_token=False,
                        map_num_proc=32):
     """Post-process the dataset loaded from the Hugging Face Hub, or a local
     dataset.
@@ -275,6 +282,9 @@ def process_hf_dataset(dataset,
         with_image_token: Whether to convert DEFAULT_IMAGE_TOKEN to
             IMAGE_TOKEN_INDEX. Typically set it to True during the training
             of VLM.
+        with_audio_token: Whether to convert DEFAULT_AUDIO_TOKEN to
+            LLAST_AUDIO_TOKEN_INDEX. Typically set it to True during the
+            training of SLM.
         map_num_proc: Max number of processes when mapping the dataset.
     """
     kwargs = dict(
@@ -293,6 +303,7 @@ def process_hf_dataset(dataset,
         use_varlen_attn=use_varlen_attn,
         input_ids_with_output=input_ids_with_output,
         with_image_token=with_image_token,
+        with_audio_token=with_audio_token,
         map_num_proc=map_num_proc)
     if not (dist.is_available() and dist.is_initialized()):
         return process(**kwargs)
diff --git a/xtuner/dataset/llast.py b/xtuner/dataset/llast.py
new file mode 100644
index 000000000..de54f8fef
--- /dev/null
+++ b/xtuner/dataset/llast.py
@@ -0,0 +1,492 @@
+# Copyright (c) LLaST. All rights reserved.
+# Copyright (c) OpenMMLab. All rights reserved.
+import csv
+import logging
+import os
+from typing import List, Optional
+
+import torch
+import whisper
+from datasets import Dataset as HFDataset
+from datasets import DatasetDict, load_from_disk
+from mmengine import print_log
+from torch.utils.data import Dataset
+from tqdm import tqdm
+from transformers import PreTrainedModel
+
+from xtuner.dataset.huggingface import process_hf_dataset
+from xtuner.utils import IGNORE_INDEX, LLAST_AUDIO_TOKEN_INDEX
+
+# Language Mapping
+LANG_DICT = {
+    'English': 'en',
+    'French': 'fr',
+    'Spanish': 'es',
+    'Chinese': 'zh-CN',
+    'German': 'de',
+    'Japanese': 'ja',
+    'Catalan': 'ca',
+    'Italian': 'it',
+    'Russian': 'ru',
+    'Portuguese': 'pt',
+    'Persian': 'fa',
+    'Estonian': 'et',
+    'Mongolian': 'mn',
+    'Dutch': 'nl',
+    'Turkish': 'tr',
+    'Arabic': 'ar',
+    'Swedish': 'sv-SE',
+    'Latvian': 'lv',
+    'Slovenian': 'sl',
+    'Tamil': 'ta',
+    'Indonesian': 'id',
+    'Welsh': 'cy'
+}
+
+SIM_LANG_DICT = {v: k for k, v in LANG_DICT.items()}
+
+device = 'cuda' if torch.cuda.device_count() > 0 else 'cpu'
+
+
+def convert_data(
+    data_path,
+    mode='s2t',
+    src_lang='French',
+    tgt_lang='English',
+    check_audio_path=False,
+    cv_dir='',
+    audio_folder='',
+    postfix='',
+):
+    """
+    Args:
+        mode (str):
+            's2t': speech-to-text translation
+            's2s': speech-to-speech translation
+            'asr': speech-to-text recognition
+        src_lang (str):
+        tgt_lang (str):
+
+    Return:
+        output_list (List)
+        ids_list (List)
+    """
+
+    assert src_lang in list(
+        LANG_DICT.keys()), 'src_languge: {} is not supported currently.'
+    assert tgt_lang in list(
+        LANG_DICT.keys()), 'tgt_language: {} is not supported currently.'
+
+    with open(data_path) as f:
+        reader = csv.DictReader(
+            f,
+            delimiter='\t',
+            quotechar=None,
+            doublequote=False,
+            lineterminator='\n',
+            quoting=csv.QUOTE_NONE,
+        )
+        raw_data = [dict(e) for e in reader]
+
+    output_list = []
+    ids_list = []
+    for item in raw_data:
+        tgt_lang_text = item['translation']
+        src_lang_text = item['sentence']
+
+        if mode == 's2t':
+            conv = {
+                'input':
+                '<audio>\nPlease translate the {} sentence into {}.'.format(
+                    src_lang, tgt_lang),
+                'output':
+                tgt_lang_text
+            }
+        elif mode == 'asr':
+            conv = {
+                'input':
+                '<audio>\nPlease transcribe the {} sentence into {}.'.format(
+                    src_lang, src_lang),
+                'output':
+                src_lang_text
+            }
+        item['id'] = item['path'].split('.')[0]
+        item_processed = dict(
+            conversation=[conv],
+            tgt_lang_text=tgt_lang_text,
+            src_lang_text=src_lang_text,
+            ids=item['id'],
+            src_lang=LANG_DICT[src_lang])
+
+        if check_audio_path:
+            wav_path = os.path.join(cv_dir, LANG_DICT[src_lang], audio_folder,
+                                    item['id'] + postfix)
+            if not os.path.exists(wav_path):
+                print_log(f'Wav:{wav_path} not exists', logger='current')
+                continue
+
+        output_list.append(item_processed)
+        ids_list.append(item['id'])
+
+    return output_list, ids_list
+
+
+def prepare_inputs_labels_for_llast(
+        llm: PreTrainedModel,
+        input_ids: torch.LongTensor = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        labels: Optional[torch.LongTensor] = None,
+        audio_tokens: Optional[torch.FloatTensor] = None,
+        audio_lens: Optional[torch.FloatTensor] = None,
+        audio_mask: Optional[torch.FloatTensor] = None):
+    if audio_tokens is None:
+        return {
+            'input_ids': input_ids,
+            'position_ids': position_ids,
+            'attention_mask': attention_mask,
+            'past_key_values': past_key_values,
+            'inputs_embeds': None,
+            'labels': labels
+        }
+    _labels = labels
+    _position_ids = position_ids
+    _attention_mask = attention_mask
+    if attention_mask is None:
+        attention_mask = torch.ones_like(input_ids, dtype=torch.bool)
+    else:
+        attention_mask = attention_mask.bool()
+    if position_ids is None:
+        position_ids = torch.arange(
+            0, input_ids.shape[1], dtype=torch.long, device=input_ids.device)
+    if labels is None:
+        labels = torch.full_like(input_ids, IGNORE_INDEX)
+
+    # remove the padding using attention_mask -- TODO: double check
+    input_ids = [
+        cur_input_ids[cur_attention_mask]
+        for cur_input_ids, cur_attention_mask in zip(input_ids, attention_mask)
+    ]
+    labels = [
+        cur_labels[cur_attention_mask]
+        for cur_labels, cur_attention_mask in zip(labels, attention_mask)
+    ]
+
+    new_inputs_embeds = []
+    new_labels = []
+    cur_audio_idx = 0
+    for batch_idx, cur_input_ids in enumerate(input_ids):
+        num_audios = (cur_input_ids == LLAST_AUDIO_TOKEN_INDEX).sum()
+        if num_audios == 0:
+            cur_audio_tokens = audio_tokens[cur_audio_idx]
+            cur_inputs_embeds_1 = llm.get_input_embeddings()(cur_input_ids)
+            cur_inputs_embeds = torch.cat(
+                [cur_inputs_embeds_1, cur_audio_tokens[0:0]], dim=0)
+            new_inputs_embeds.append(cur_inputs_embeds)
+            new_labels.append(labels[batch_idx])
+            cur_audio_idx += 1
+            continue
+
+        audio_token_indices = [-1] + torch.where(
+            cur_input_ids == LLAST_AUDIO_TOKEN_INDEX)[0].tolist() + [
+                cur_input_ids.shape[0]
+            ]
+        cur_input_ids_noaudio = []
+        cur_labels = labels[batch_idx]
+        cur_labels_noaudio = []
+        for i in range(len(audio_token_indices) - 1):
+            cur_input_ids_noaudio.append(
+                cur_input_ids[audio_token_indices[i] +
+                              1:audio_token_indices[i + 1]])
+            cur_labels_noaudio.append(cur_labels[audio_token_indices[i] +
+                                                 1:audio_token_indices[i + 1]])
+        split_sizes = [x.shape[0] for x in cur_labels_noaudio]
+        cur_inputs_embeds = llm.get_input_embeddings()(
+            torch.cat(cur_input_ids_noaudio))
+        cur_inputs_embeds_no_audio = torch.split(
+            cur_inputs_embeds, split_sizes, dim=0)
+
+        cur_new_inputs_embeds = []
+        cur_new_labels = []
+
+        for i in range(num_audios + 1):
+            cur_new_inputs_embeds.append(cur_inputs_embeds_no_audio[i])
+            cur_new_labels.append(cur_labels_noaudio[i])
+            if i == 0:
+                prefix_len = cur_inputs_embeds_no_audio[i].shape[0]
+            if i < num_audios:
+                cur_audio_tokens = audio_tokens[cur_audio_idx]
+                cur_audio_idx += 1
+                cur_new_inputs_embeds.append(cur_audio_tokens)
+                cur_new_labels.append(
+                    torch.full((cur_audio_tokens.shape[0], ),
+                               IGNORE_INDEX,
+                               device=cur_labels.device,
+                               dtype=cur_labels.dtype))
+
+        cur_new_inputs_embeds = torch.cat(cur_new_inputs_embeds)
+        cur_new_labels = torch.cat(cur_new_labels)
+
+        new_inputs_embeds.append(cur_new_inputs_embeds)
+        new_labels.append(cur_new_labels)
+
+    # Combine them
+    max_len = max(x.shape[0] for x in new_inputs_embeds)
+    batch_size = len(new_inputs_embeds)
+
+    new_inputs_embeds_padded = []
+    new_labels_padded = torch.full((batch_size, max_len),
+                                   IGNORE_INDEX,
+                                   dtype=new_labels[0].dtype,
+                                   device=new_labels[0].device)
+    attention_mask = torch.zeros((batch_size, max_len),
+                                 dtype=attention_mask.dtype,
+                                 device=attention_mask.device)
+    position_ids = torch.zeros((batch_size, max_len),
+                               dtype=position_ids.dtype,
+                               device=position_ids.device)
+
+    # remain debug
+    if audio_lens is None and audio_tokens.shape[0] == 1:
+        audio_lens = [audio_tokens.shape[1]]
+    max_audio_len = max(audio_lens)
+    for i, (cur_new_embed, cur_new_labels, audio_len) in enumerate(
+            zip(new_inputs_embeds, new_labels, audio_lens)):
+        cur_len = cur_new_embed.shape[0]
+        new_inputs_embeds_padded.append(
+            torch.cat((cur_new_embed,
+                       torch.zeros((max_len - cur_len, cur_new_embed.shape[1]),
+                                   dtype=cur_new_embed.dtype,
+                                   device=cur_new_embed.device)),
+                      dim=0))
+        if cur_len > 0:
+            new_labels_padded[i, :cur_len] = cur_new_labels
+            attention_mask[i, :cur_len] = True
+            attention_mask[i, (prefix_len + audio_len):(prefix_len +
+                                                        max_audio_len)] = False
+            position_ids[i, :cur_len] = torch.arange(
+                0,
+                cur_len,
+                dtype=position_ids.dtype,
+                device=position_ids.device)
+
+    new_inputs_embeds = torch.stack(new_inputs_embeds_padded, dim=0)
+
+    if _labels is None:
+        new_labels = None
+    else:
+        new_labels = new_labels_padded
+
+    if _attention_mask is None:
+        attention_mask = None
+    else:
+        attention_mask = attention_mask.to(dtype=_attention_mask.dtype)
+
+    if _position_ids is None:
+        position_ids = None
+
+    return {
+        'input_ids': None,
+        'position_ids': position_ids,
+        'attention_mask': attention_mask,
+        'past_key_values': past_key_values,
+        'inputs_embeds': new_inputs_embeds,
+        'labels': new_labels
+    }
+
+
+class LLaSTDataset(Dataset):
+    """Dataset for LLaST.
+
+    Args:
+        en2x_sample_step (int): sample step of en2x data (1 means, no sampling,
+            10 means sampling 1/10 data)
+    """
+
+    def __init__(self,
+                 tsv_dir,
+                 tokenizer,
+                 offline_processed_text_folder=None,
+                 max_dataset_length=None,
+                 dataset_map_fn=None,
+                 template_map_fn=None,
+                 max_length=2048,
+                 input_ids_with_output=True,
+                 audio_folder='clips',
+                 debug=False,
+                 split='train',
+                 cv_dir=None,
+                 with_asr=True,
+                 en2x_sample_step=1,
+                 check_audio_path=False,
+                 x2en=['fr', 'es'],
+                 en2x=['ja', 'de']):
+
+        super().__init__()
+
+        assert offline_processed_text_folder or (tsv_dir and tokenizer)
+        if offline_processed_text_folder and tsv_dir:
+            print_log(
+                'Both `offline_processed_text_folder` and '
+                '`tsv_dir` are set, and we load dataset from'
+                '`offline_processed_text_folder` '
+                f'({offline_processed_text_folder})',
+                logger='current',
+                level=logging.WARNING)
+
+        self.cv_dir = cv_dir
+        self.postfix = '.wav'
+        self.audio_folder = audio_folder
+        if offline_processed_text_folder is not None and os.path.exists(
+                offline_processed_text_folder):
+            self.text_data = load_from_disk(offline_processed_text_folder)
+        else:
+            raw_data, ids = self.get_data_ids(
+                tsv_dir=tsv_dir,
+                split=split,
+                with_asr=with_asr,
+                check_audio_path=check_audio_path,
+                cv_dir=cv_dir,
+                audio_folder=audio_folder,
+                en2x_sample_step=en2x_sample_step,
+                x2en=x2en,
+                en2x=en2x)
+
+            if debug:
+                debug_length = int(len(raw_data) * 0.01)
+                raw_data, ids = raw_data[:debug_length], ids[:debug_length]
+
+            raw_data = DatasetDict({split: HFDataset.from_list(raw_data)})
+
+            self.text_data = process_hf_dataset(
+                # TODO: need to merge latest xtuner
+                # broadcast_from_master=False,
+                dataset=raw_data,
+                tokenizer=tokenizer,
+                max_length=max_length,
+                dataset_map_fn=dataset_map_fn,
+                template_map_fn=template_map_fn,
+                split=split,
+                max_dataset_length=max_dataset_length,
+                remove_unused_columns=False,
+                pack_to_max_length=False,
+                with_audio_token=True,
+                input_ids_with_output=input_ids_with_output)
+
+    def get_data_ids(self, tsv_dir, split, with_asr, check_audio_path, cv_dir,
+                     audio_folder, en2x_sample_step, x2en, en2x):
+
+        raw_data = []
+        ids = []
+
+        for src in x2en:
+            data_path = os.path.join(tsv_dir,
+                                     f'covost_v2.{src}_en.{split}.tsv')
+            x2en_raw_data, x2en_ids = convert_data(
+                data_path,
+                src_lang=SIM_LANG_DICT[src],
+                tgt_lang=SIM_LANG_DICT['en'],
+                mode='s2t',
+                check_audio_path=check_audio_path,
+                cv_dir=cv_dir,
+                audio_folder=audio_folder,
+                postfix=self.postfix,
+            )
+            print_log(f'{src} to en:{len(x2en_raw_data)}', logger='current')
+            raw_data += x2en_raw_data
+            ids += x2en_ids
+            # augement with asr data
+            if split == 'train' and with_asr:
+                print_log('*' * 30, logger='current')
+                print_log(
+                    'split is train, augment data with asr data',
+                    logger='current')
+                raw_data_asr, ids_asr = convert_data(
+                    data_path, src_lang=SIM_LANG_DICT[src], mode='asr')
+                raw_data += raw_data_asr
+                ids += ids_asr
+
+        for tgt in en2x:
+            data_path = os.path.join(tsv_dir,
+                                     f'covost_v2.en_{tgt}.{split}.tsv')
+
+            en2x_raw_data, en2x_ids = convert_data(
+                data_path,
+                src_lang=SIM_LANG_DICT['en'],
+                tgt_lang=SIM_LANG_DICT[tgt],
+                mode='s2t',
+                check_audio_path=check_audio_path,
+                cv_dir=cv_dir,
+                audio_folder=audio_folder,
+                postfix=self.postfix)
+
+            print_log(f'en to {tgt}:{len(en2x_raw_data)}', logger='current')
+            # use sampling
+            print_log(f'sampling step:{en2x_sample_step}', logger='current')
+            assert en2x_sample_step < len(en2x_raw_data)
+            en2x_raw_data, en2x_ids = en2x_raw_data[::en2x_sample_step], \
+                en2x_ids[::en2x_sample_step]
+            print_log(
+                f'after sampled, en to {tgt}:{len(en2x_raw_data)}',
+                logger='current')
+            raw_data += en2x_raw_data
+            ids += en2x_ids
+
+        # augement with asr data of English
+        if split == 'train' and with_asr and len(en2x) > 0:
+            print_log(
+                'split is train, augment data with asr data', logger='current')
+            raw_data_asr, ids_asr = convert_data(
+                data_path, src_lang=SIM_LANG_DICT['en'], mode='asr')
+            raw_data += raw_data_asr
+            ids += ids_asr
+
+        return raw_data, ids
+
+    @property
+    def modality_length(self):
+        length_list = []
+        for data_dict in self.text_data:
+            cur_len = len(data_dict['input_ids'])
+            if data_dict.get('audio', None) is None:
+                cur_len = -cur_len
+            length_list.append(cur_len)
+        return length_list
+
+    def __len__(self):
+        return len(self.text_data)
+
+    def get_whisper_features(self, cv_dir, ids, postfix='.mp3.wav'):
+        self.whisper_feats = {}
+        print_log('extract whisper fetures', logger='current')
+        for item_id in tqdm(ids):
+            item_id = item_id.split('.')[0]
+            wav_path = os.path.join(cv_dir, self.audio_folder,
+                                    item_id + postfix)
+            feats = self.extract_whisper_features(wav_path)
+            self.whisper_feats[item_id] = feats
+
+    def __getitem__(self, index):
+        data_dict = self.text_data[index]
+
+        wav_path = os.path.join(self.cv_dir, data_dict['src_lang'],
+                                self.audio_folder,
+                                data_dict['ids'].split('.')[0] + self.postfix)
+
+        audio = whisper.load_audio(str(wav_path))
+        audio = torch.from_numpy(audio)
+        audio_len = audio.shape[0]
+        audio = whisper.pad_or_trim(audio)
+        max_audio_len = audio.shape[0]
+
+        # (80, 3000)
+        with torch.no_grad():
+            mel = whisper.log_mel_spectrogram(audio)
+
+        max_mel_len = mel.shape[1]
+        mel_features_len = int(audio_len / max_audio_len * max_mel_len)
+        features_len = mel_features_len // 2
+        data_dict['audio_tokens'] = mel
+        data_dict['audio_lens'] = features_len
+        return data_dict
diff --git a/xtuner/dataset/utils.py b/xtuner/dataset/utils.py
index 84336ddb2..967cf649e 100644
--- a/xtuner/dataset/utils.py
+++ b/xtuner/dataset/utils.py
@@ -9,7 +9,9 @@
 import requests
 from PIL import Image
 
-from xtuner.utils import DEFAULT_IMAGE_TOKEN, IGNORE_INDEX, IMAGE_TOKEN_INDEX
+from xtuner.utils import (DEFAULT_AUDIO_TOKEN, DEFAULT_IMAGE_TOKEN,
+                          IGNORE_INDEX, IMAGE_TOKEN_INDEX,
+                          LLAST_AUDIO_TOKEN_INDEX)
 
 
 def get_bos_eos_token_ids(tokenizer):
@@ -37,7 +39,8 @@ def encode_fn(example,
               tokenizer,
               max_length,
               input_ids_with_output=True,
-              with_image_token=False):
+              with_image_token=False,
+              with_audio_token=False):
     """We only support the following three scenarios:
 
     1. Incremental pretraining dataset.
@@ -88,6 +91,19 @@ def encode_fn(example,
                 input_encode.extend(cur_chunk_encode)
                 if idx != len(chunk_encode) - 1:
                     input_encode.append(IMAGE_TOKEN_INDEX)
+        elif DEFAULT_AUDIO_TOKEN in input and with_audio_token:
+            # for adding audio token
+            chunk_encode = [
+                tokenizer.encode(chunk, add_special_tokens=False)
+                for chunk in input.split(DEFAULT_AUDIO_TOKEN)
+            ]
+
+            assert len(chunk_encode) == 2
+            input_encode = []
+            for idx, cur_chunk_encode in enumerate(chunk_encode):
+                input_encode.extend(cur_chunk_encode)
+                if idx != len(chunk_encode) - 1:
+                    input_encode.append(LLAST_AUDIO_TOKEN_INDEX)
         else:
             input_encode = tokenizer.encode(input, add_special_tokens=False)
         if next_needs_bos_token:
diff --git a/xtuner/engine/__init__.py b/xtuner/engine/__init__.py
index 4f50972ea..3b6f2f192 100644
--- a/xtuner/engine/__init__.py
+++ b/xtuner/engine/__init__.py
@@ -2,9 +2,10 @@
 from ._strategy import DeepSpeedStrategy
 from .hooks import (DatasetInfoHook, EvaluateChatHook, ThroughputHook,
                     VarlenAttnArgsToMessageHubHook)
-from .runner import TrainLoop
+from .runner import LLaSTTestLoop, TrainLoop
 
 __all__ = [
     'EvaluateChatHook', 'DatasetInfoHook', 'ThroughputHook',
-    'VarlenAttnArgsToMessageHubHook', 'DeepSpeedStrategy', 'TrainLoop'
+    'VarlenAttnArgsToMessageHubHook', 'DeepSpeedStrategy', 'TrainLoop',
+    'LLaSTTestLoop'
 ]
diff --git a/xtuner/engine/runner/__init__.py b/xtuner/engine/runner/__init__.py
index d8d1c582b..20d3c9736 100644
--- a/xtuner/engine/runner/__init__.py
+++ b/xtuner/engine/runner/__init__.py
@@ -1,4 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
+from .llast_loops import LLaSTTestLoop
 from .loops import TrainLoop
 
-__all__ = ['TrainLoop']
+__all__ = ['TrainLoop', 'LLaSTTestLoop']
diff --git a/xtuner/engine/runner/llast_loops.py b/xtuner/engine/runner/llast_loops.py
new file mode 100644
index 000000000..762539cd0
--- /dev/null
+++ b/xtuner/engine/runner/llast_loops.py
@@ -0,0 +1,166 @@
+# Copyright (c) LLaST. All rights reserved.
+# Copyright (c) OpenMMLab. All rights reserved.
+from typing import Dict, List, Sequence
+
+import torch
+from mmengine.evaluator import Evaluator
+from mmengine.model import is_model_wrapper
+from mmengine.registry import LOOPS
+from mmengine.runner import TestLoop, autocast
+from mmengine.utils.misc import get_object_from_string
+from torch.utils.data import DataLoader
+from transformers import GenerationConfig, StoppingCriteriaList
+
+from xtuner.dataset.llast import prepare_inputs_labels_for_llast
+from xtuner.registry import BUILDER
+from xtuner.utils import StopWordStoppingCriteria
+
+
+def get_stop_criteria(
+    tokenizer,
+    stop_words=[],
+):
+    stop_criteria = StoppingCriteriaList()
+    for word in stop_words:
+        stop_criteria.append(StopWordStoppingCriteria(tokenizer, word))
+    return stop_criteria
+
+
+@LOOPS.register_module()
+class LLaSTTestLoop(TestLoop):
+
+    def __init__(self,
+                 runner,
+                 tokenizer,
+                 dataloader: DataLoader | Dict,
+                 evaluator: Evaluator | Dict | List,
+                 fp16: bool = False,
+                 system='',
+                 prompt_template=None,
+                 max_new_tokens=256,
+                 num_beams=1,
+                 do_sample=True,
+                 stop_word=None,
+                 end_str=None):
+        super().__init__(runner, dataloader, evaluator, fp16)
+        if prompt_template is None:
+            instruction = '{input}'
+        else:
+            if isinstance(prompt_template, str):  # for resume
+                prompt_template = get_object_from_string(prompt_template)
+            instruction = prompt_template.get('INSTRUCTION', '{input}')
+            if system != '':
+                system = prompt_template.get(
+                    'SYSTEM', '{system}\n').format(system=system)
+        self.instruction = instruction
+        self.system = system
+        self.max_new_tokens = max_new_tokens
+        self.tokenizer = BUILDER.build(tokenizer)
+        # default generation config
+        self.gen_config = GenerationConfig(
+            num_beams=num_beams,
+            max_new_tokens=max_new_tokens,
+            do_sample=do_sample,
+            temperature=0.1,
+            top_p=0.75,
+            top_k=40,
+            eos_token_id=self.tokenizer.eos_token_id,
+            pad_token_id=self.tokenizer.pad_token_id
+            if self.tokenizer.pad_token_id is not None else
+            self.tokenizer.eos_token_id,
+        )
+        if stop_word is not None:
+            self.stop_criteria = get_stop_criteria(
+                tokenizer=self.tokenizer, stop_words=stop_word)
+        else:
+            self.stop_criteria = StoppingCriteriaList()
+        self.end_str = end_str
+
+    @torch.no_grad()
+    def run_iter(self, idx, data_batch: Sequence[dict]) -> None:
+        """Iterate one mini-batch.
+
+        Args:
+            data_batch (Sequence[dict]): Batch of data from dataloader.
+        """
+        self.runner.call_hook(
+            'before_test_iter', batch_idx=idx, data_batch=data_batch)
+        # predictions should be sequence of BaseDataElement
+        if self.fp16:
+            if is_model_wrapper(self.runner.model):
+                self.runner.model.module = self.runner.model.module.to(
+                    torch.float16)
+            else:
+                self.runner.model = self.runner.model.to(torch.float16)
+
+        with autocast(enabled=self.fp16, dtype=torch.float16):
+            if is_model_wrapper(self.runner.model):
+                data_preprocessor = self.runner.model.module.data_preprocessor  # noqa: E501
+                audio_data_dtype = self.runner.model.module.speech_encoder.encoder.conv1.weight.dtype  # noqa: E501
+                llm_data_dtype = self.runner.model.module.projector.model[
+                    0].weight.dtype
+                projector = self.runner.model.module.projector
+                speech_encoder = self.runner.model.module.speech_encoder
+                llm = self.runner.model.module.llm
+                generate_fn = self.runner.model.module.generate
+                decoder_start_token_id = self.runner.model.module.speech_encoder.config.decoder_start_token_id  # noqa: E501
+            else:
+                data_preprocessor = self.runner.model.data_preprocessor
+                audio_data_dtype = self.runner.model.speech_encoder.encoder.conv1.weight.dtype  # noqa: E501
+                llm_data_dtype = self.runner.model.projector.model[
+                    0].weight.dtype
+                speech_encoder = self.runner.model.speech_encoder
+                projector = self.runner.model.projector
+                llm = self.runner.model.llm
+                generate_fn = self.runner.model.generate
+                decoder_start_token_id = self.runner.model.speech_encoder.config.decoder_start_token_id  # noqa: E501
+
+            data_batch = data_preprocessor(data_batch, False)
+
+            data = data_batch['data']
+            data['audio_tokens'] = data['audio_tokens'].to(audio_data_dtype)
+            batch_size = data['audio_tokens'].shape[0]
+            decoder_input_ids = torch.tensor([[1] * batch_size
+                                              ]) * decoder_start_token_id
+            audio_outputs = speech_encoder(
+                data['audio_tokens'],
+                decoder_input_ids=decoder_input_ids.to(
+                    data['audio_tokens'].device),
+                output_hidden_states=True).encoder_last_hidden_state
+
+            audio_outputs = audio_outputs.to(llm_data_dtype)
+            audio_outputs = audio_outputs[:, :max(data['audio_lens']), :]
+
+            audio_tokens = projector(audio_outputs)
+            data['audio_tokens'] = audio_tokens
+
+            mm_inputs = prepare_inputs_labels_for_llast(
+                llm=llm,
+                input_ids=data['input_ids'],
+                audio_lens=data['audio_lens'],
+                audio_tokens=audio_tokens)
+
+            # dtype = self.runner.model.llm.dtype
+            mm_inputs['inputs_embeds'] = mm_inputs['inputs_embeds'].to(
+                llm_data_dtype)
+            generation_output = generate_fn(
+                **mm_inputs,
+                max_new_tokens=self.max_new_tokens,
+                generation_config=self.gen_config,
+                bos_token_id=self.tokenizer.bos_token_id,
+                stopping_criteria=self.stop_criteria)
+
+            generations = self.tokenizer.batch_decode(
+                generation_output, skip_special_tokens=True)
+
+            if self.end_str:
+                generations = [
+                    item.split(self.end_str)[0] for item in generations
+                ]
+
+        self.evaluator.process(data_samples=generations, data_batch=data_batch)
+        self.runner.call_hook(
+            'after_test_iter',
+            batch_idx=idx,
+            data_batch=data_batch,
+            outputs=generations)
diff --git a/xtuner/evaluation/metrics/__init__.py b/xtuner/evaluation/metrics/__init__.py
index f3efc80fd..c80115337 100644
--- a/xtuner/evaluation/metrics/__init__.py
+++ b/xtuner/evaluation/metrics/__init__.py
@@ -1,4 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 from .mmlu_metric import MMLUMetric
+from .sacrebleu_metric import SacreBLEUMetric
 
-__all__ = ['MMLUMetric']
+__all__ = ['MMLUMetric', 'SacreBLEUMetric']
diff --git a/xtuner/evaluation/metrics/sacrebleu.py b/xtuner/evaluation/metrics/sacrebleu.py
new file mode 100644
index 000000000..ae1a6cf53
--- /dev/null
+++ b/xtuner/evaluation/metrics/sacrebleu.py
@@ -0,0 +1,182 @@
+# Copyright 2020 The HuggingFace Evaluate Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""SACREBLEU metric."""
+
+import datasets
+import evaluate
+import sacrebleu as scb
+from packaging import version
+
+_CITATION = """\
+@inproceedings{post-2018-call,
+    title = "A Call for Clarity in Reporting {BLEU} Scores",
+    author = "Post, Matt",
+    booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers",
+    month = oct,
+    year = "2018",
+    address = "Belgium, Brussels",
+    publisher = "Association for Computational Linguistics",
+    url = "https://www.aclweb.org/anthology/W18-6319",
+    pages = "186--191",
+}
+"""
+
+_DESCRIPTION = """\
+SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores.
+Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text.
+It also knows all the standard test sets and handles downloading, processing, and tokenization for you.
+
+See the [README.md] file at https://github.com/mjpost/sacreBLEU for more information.
+"""
+
+_KWARGS_DESCRIPTION = """
+Produces BLEU scores along with its sufficient statistics
+from a source against one or more references.
+
+Args:
+    predictions (`list` of `str`): list of translations to score. Each translation should be tokenized into a list of tokens.
+    references (`list` of `list` of `str`): A list of lists of references. The contents of the first sub-list are the references for the first prediction, the contents of the second sub-list are for the second prediction, etc. Note that there must be the same number of references for each prediction (i.e. all sub-lists must be of the same length).
+    smooth_method (`str`): The smoothing method to use, defaults to `'exp'`. Possible values are:
+        - `'none'`: no smoothing
+        - `'floor'`: increment zero counts
+        - `'add-k'`: increment num/denom by k for n>1
+        - `'exp'`: exponential decay
+    smooth_value (`float`): The smoothing value. Only valid when `smooth_method='floor'` (in which case `smooth_value` defaults to `0.1`) or `smooth_method='add-k'` (in which case `smooth_value` defaults to `1`).
+    tokenize (`str`): Tokenization method to use for BLEU. If not provided, defaults to `'zh'` for Chinese, `'ja-mecab'` for Japanese and `'13a'` (mteval) otherwise. Possible values are:
+        - `'none'`: No tokenization.
+        - `'zh'`: Chinese tokenization.
+        - `'13a'`: mimics the `mteval-v13a` script from Moses.
+        - `'intl'`: International tokenization, mimics the `mteval-v14` script from Moses
+        - `'char'`: Language-agnostic character-level tokenization.
+        - `'ja-mecab'`: Japanese tokenization. Uses the [MeCab tokenizer](https://pypi.org/project/mecab-python3).
+    lowercase (`bool`): If `True`, lowercases the input, enabling case-insensitivity. Defaults to `False`.
+    force (`bool`): If `True`, insists that your tokenized input is actually detokenized. Defaults to `False`.
+    use_effective_order (`bool`): If `True`, stops including n-gram orders for which precision is 0. This should be `True`, if sentence-level BLEU will be computed. Defaults to `False`.
+
+Returns:
+    'score': BLEU score,
+    'counts': Counts,
+    'totals': Totals,
+    'precisions': Precisions,
+    'bp': Brevity penalty,
+    'sys_len': predictions length,
+    'ref_len': reference length,
+
+Examples:
+
+    Example 1:
+        >>> predictions = ["hello there general kenobi", "foo bar foobar"]
+        >>> references = [["hello there general kenobi", "hello there !"], ["foo bar foobar", "foo bar foobar"]]
+        >>> sacrebleu = evaluate.load("sacrebleu")
+        >>> results = sacrebleu.compute(predictions=predictions, references=references)
+        >>> print(list(results.keys()))
+        ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']
+        >>> print(round(results["score"], 1))
+        100.0
+
+    Example 2:
+        >>> predictions = ["hello there general kenobi",
+        ...                 "on our way to ankh morpork"]
+        >>> references = [["hello there general kenobi", "hello there !"],
+        ...                 ["goodbye ankh morpork", "ankh morpork"]]
+        >>> sacrebleu = evaluate.load("sacrebleu")
+        >>> results = sacrebleu.compute(predictions=predictions,
+        ...                             references=references)
+        >>> print(list(results.keys()))
+        ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']
+        >>> print(round(results["score"], 1))
+        39.8
+"""
+
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION,
+                                                _KWARGS_DESCRIPTION)
+class Sacrebleu(evaluate.Metric):
+
+    def _info(self):
+        if version.parse(scb.__version__) < version.parse('1.4.12'):
+            raise ImportWarning(
+                "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n"
+                'You can install it with `pip install "sacrebleu>=1.4.12"`.')
+        return evaluate.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            homepage='https://github.com/mjpost/sacreBLEU',
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=[
+                datasets.Features({
+                    'predictions':
+                    datasets.Value('string', id='sequence'),
+                    'references':
+                    datasets.Sequence(
+                        datasets.Value('string', id='sequence'),
+                        id='references'),
+                }),
+                datasets.Features({
+                    'predictions':
+                    datasets.Value('string', id='sequence'),
+                    'references':
+                    datasets.Value('string', id='sequence'),
+                }),
+            ],
+            codebase_urls=['https://github.com/mjpost/sacreBLEU'],
+            reference_urls=[
+                'https://github.com/mjpost/sacreBLEU',
+                'https://en.wikipedia.org/wiki/BLEU',
+                'https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213',
+            ],
+        )
+
+    def _compute(
+        self,
+        predictions,
+        references,
+        smooth_method='exp',
+        smooth_value=None,
+        force=False,
+        lowercase=False,
+        tokenize=None,
+        use_effective_order=False,
+    ):
+        # if only one reference is provided make sure we still use list of lists
+        if isinstance(references[0], str):
+            references = [[ref] for ref in references]
+
+        references_per_prediction = len(references[0])
+        if any(len(refs) != references_per_prediction for refs in references):
+            raise ValueError(
+                'Sacrebleu requires the same number of references for each prediction'
+            )
+        transformed_references = [[refs[i] for refs in references]
+                                  for i in range(references_per_prediction)]
+        output = scb.corpus_bleu(
+            predictions,
+            transformed_references,
+            smooth_method=smooth_method,
+            smooth_value=smooth_value,
+            force=force,
+            lowercase=lowercase,
+            use_effective_order=use_effective_order,
+            **(dict(tokenize=tokenize) if tokenize else {}),
+        )
+        output_dict = {
+            'score': output.score,
+            'counts': output.counts,
+            'totals': output.totals,
+            'precisions': output.precisions,
+            'bp': output.bp,
+            'sys_len': output.sys_len,
+            'ref_len': output.ref_len,
+        }
+        return output_dict
diff --git a/xtuner/evaluation/metrics/sacrebleu_metric.py b/xtuner/evaluation/metrics/sacrebleu_metric.py
new file mode 100644
index 000000000..a0e7363c2
--- /dev/null
+++ b/xtuner/evaluation/metrics/sacrebleu_metric.py
@@ -0,0 +1,112 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import json
+import os
+from typing import Any, Sequence
+
+import evaluate
+import numpy as np
+from mmengine.evaluator import BaseMetric
+from mmengine.logging import print_log
+from rich.console import Console
+from rich.table import Table
+
+import xtuner
+from xtuner.registry import BUILDER
+
+
+class SacreBLEUMetric(BaseMetric):
+
+    def __init__(self,
+                 tokenizer,
+                 dump_dir='./',
+                 epoch_num='',
+                 *args,
+                 **kwargs):
+        super().__init__(*args, **kwargs)
+        tokenizer = BUILDER.build(tokenizer)
+        xtuner_path = os.path.dirname(xtuner.__file__)
+        self.sacrebleu = evaluate.load(
+            os.path.join(xtuner_path, 'evaluation/metrics/sacrebleu.py'))
+        self.dump_dir = dump_dir
+        self.epoch_num = epoch_num
+
+    @staticmethod
+    def accuracy(preds, gts):
+        """Computes the accuracy for preds and gts."""
+        correct = [1 if pred == gt else 0 for pred, gt in zip(preds, gts)]
+        acc = np.mean(correct) * 100
+        return acc
+
+    def process(self, data_batch: Any, generations: Sequence[dict]) -> None:
+        """Process one batch of data samples and predictions. The processed
+        results should be stored in ``self.results``, which will be used to
+        compute the metrics when all batches have been processed.
+
+        Args:
+            data_batch (Any): A batch of data from the dataloader.
+            data_samples (Sequence[dict]): A batch of outputs from
+                the model.
+        """
+
+        for predict, data_sample in zip(generations,
+                                        data_batch['data_samples']):
+            predict = predict
+            golden_text = data_sample['tgt_lang_text']
+
+            print('---' * 10 + 'Start' + '---' * 10)
+            print('Prediction:\n{}\nGolden:\n{}\n'.format(
+                predict, golden_text))
+            print('---' * 10 + 'End' + '---' * 10)
+            self.results.append(
+                dict(predictions=predict, references=golden_text))
+            tmp_file_path = os.path.join(
+                self.dump_dir, f'tmp_inference_results{self.epoch_num}.json')
+            with open(tmp_file_path, 'w') as file:
+                json.dump(self.results, file, indent=4)
+            # import pdb;pdb.set_trace()
+
+    def compute_metrics(self, results: list, tokenize='13a') -> dict:
+        """Compute the metrics from processed results.
+
+        Args:
+            results (list): The processed results of each batch.
+
+        Returns:
+            dict: The computed metrics. The keys are the names of the metrics,
+            and the values are corresponding results.
+        """
+        if 'en_zh' in self.epoch_num:
+            tokenize = 'zh'
+        elif 'en_ja' in self.epoch_num:
+            tokenize = 'ja-mecab'
+
+        file_path = os.path.join(self.dump_dir,
+                                 f'inference_results{self.epoch_num}.json')
+
+        with open(file_path, 'w') as file:
+            json.dump(results, file, indent=4)
+
+        print(f'Total Length of the Prediction:{len(results)}')
+        predictions = [item['predictions'] for item in results]
+        references = [item['references'] for item in results]
+
+        # convert to lower
+        predictions = [item.lower() for item in predictions]
+        references = [item.lower() for item in references]
+
+        metric_results = self.sacrebleu.compute(
+            predictions=predictions, references=references, tokenize=tokenize)
+        self._print_results(metric_results)
+        return metric_results
+
+    def _print_results(self, table_metrics: dict) -> None:
+        table_title = ' SacreBLEU Metric '
+        table = Table(title=table_title)
+        console = Console()
+        table.add_column('Metric', justify='left')
+        table.add_column('Value', justify='right')
+        for key, value in table_metrics.items():
+            table.add_row(key, str(value))
+        with console.capture() as capture:
+            console.print(table, end='')
+        print_log('\n' + capture.get(), 'current')
diff --git a/xtuner/model/__init__.py b/xtuner/model/__init__.py
index 1b3a501d4..633969318 100644
--- a/xtuner/model/__init__.py
+++ b/xtuner/model/__init__.py
@@ -1,6 +1,7 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 from .internvl import InternVL_V1_5
+from .llast import LLaSTModel
 from .llava import LLaVAModel
 from .sft import SupervisedFinetune
 
-__all__ = ['SupervisedFinetune', 'LLaVAModel', 'InternVL_V1_5']
+__all__ = ['SupervisedFinetune', 'LLaVAModel', 'InternVL_V1_5', 'LLaSTModel']
diff --git a/xtuner/model/llast.py b/xtuner/model/llast.py
index 246fd7cde..3ed419873 100644
--- a/xtuner/model/llast.py
+++ b/xtuner/model/llast.py
@@ -1,3 +1,4 @@
+# Copyright (c) LLaST. All rights reserved.
 # Copyright (c) OpenMMLab. All rights reserved.
 from collections import OrderedDict
 
diff --git a/xtuner/tools/process_llast_data.py b/xtuner/tools/process_llast_data.py
new file mode 100644
index 000000000..ba27fccff
--- /dev/null
+++ b/xtuner/tools/process_llast_data.py
@@ -0,0 +1,33 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import warnings
+
+from mmengine import Config
+
+from xtuner.registry import BUILDER
+
+# ignore FutureWarning in hf datasets
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('config', help='config file name or path.')
+    parser.add_argument('--save-folder', help='The folder to save data order.')
+    args = parser.parse_args()
+    return args
+
+
+def build_llast_dataset(config):
+    dataset = BUILDER.build(config.train_dataloader.dataset)
+    return dataset
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    cfg = Config.fromfile(args.config)
+
+    llast_dataset = build_llast_dataset(cfg)
+    text_data = llast_dataset.text_data
+
+    text_data.save_to_disk(args.save_folder)
diff --git a/xtuner/utils/__init__.py b/xtuner/utils/__init__.py
index 6663b3225..3019c0eb2 100644
--- a/xtuner/utils/__init__.py
+++ b/xtuner/utils/__init__.py
@@ -1,6 +1,8 @@
 # Copyright (c) OpenMMLab. All rights reserved.
-from .constants import (DEFAULT_IMAGE_TOKEN, DEFAULT_PAD_TOKEN_INDEX,
-                        IGNORE_INDEX, IMAGE_TOKEN_INDEX)
+from .constants import (DEFAULT_AUDIO_TOKEN, DEFAULT_IMAGE_TOKEN,
+                        DEFAULT_PAD_TOKEN_INDEX, IGNORE_INDEX,
+                        IMAGE_TOKEN_INDEX, LLAST_AUDIO_PADDING_TOKEN_INDEX,
+                        LLAST_AUDIO_TOKEN_INDEX)
 from .handle_moe_load_and_save import (SUPPORT_MODELS, get_origin_state_dict,
                                        load_state_dict_into_model)
 from .stop_criteria import StopWordStoppingCriteria
@@ -10,5 +12,6 @@
     'IGNORE_INDEX', 'DEFAULT_PAD_TOKEN_INDEX', 'PROMPT_TEMPLATE',
     'DEFAULT_IMAGE_TOKEN', 'SYSTEM_TEMPLATE', 'StopWordStoppingCriteria',
     'IMAGE_TOKEN_INDEX', 'load_state_dict_into_model', 'get_origin_state_dict',
-    'SUPPORT_MODELS'
+    'SUPPORT_MODELS', 'LLAST_AUDIO_TOKEN_INDEX', 'DEFAULT_AUDIO_TOKEN',
+    'LLAST_AUDIO_PADDING_TOKEN_INDEX'
 ]
diff --git a/xtuner/utils/constants.py b/xtuner/utils/constants.py
index 2862c8ab5..8e951b6c9 100644
--- a/xtuner/utils/constants.py
+++ b/xtuner/utils/constants.py
@@ -3,3 +3,6 @@
 DEFAULT_PAD_TOKEN_INDEX = 0
 IMAGE_TOKEN_INDEX = -200
 DEFAULT_IMAGE_TOKEN = '<image>'
+DEFAULT_AUDIO_TOKEN = '<audio>'
+LLAST_AUDIO_TOKEN_INDEX = -300  # for LLaST
+LLAST_AUDIO_PADDING_TOKEN_INDEX = 1000  # padding token for LLaST
diff --git a/xtuner/utils/templates.py b/xtuner/utils/templates.py
index 59b472731..dbc571c53 100644
--- a/xtuner/utils/templates.py
+++ b/xtuner/utils/templates.py
@@ -44,6 +44,10 @@
             'nature.\n{system}\n<</SYS>>\n [/INST] '),
         INSTRUCTION='[INST] {input} [/INST]',
         SEP='\n'),
+    tinyllama_chat=dict(
+        SYSTEM='<|system|>\n{system}</s>\n',
+        INSTRUCTION='<|user|>\n{input}</s>\n<|assistant|></s>\n',
+        SEP='\n'),
     code_llama_chat=dict(
         SYSTEM='{system}\n', INSTRUCTION='[INST] {input} [/INST]'),
     chatglm2=dict(