Support building docker image manually in CI

.github/workflows/test.yml

@@ -7,6 +7,7 @@ on:
       - ".github/workflows/test.yml"
       - "cmake/**"
       - "src/**"
+      - "autotest/**"
       - "3rdparty/**"
       - "lmdeploy/**"
       - "requirements/**"
@@ -62,13 +63,13 @@ jobs:
       - name: Install pytorch
         run: |
           python3 -m pip cache dir
-          python3 -m pip install torch==1.13.1+cu117 torchvision==0.14.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117
+          python3 -m pip install torch==2.0.1 torchvision==0.15.2 --extra-index-url https://download.pytorch.org/whl/cu117
       - name: Build lmdeploy
         run: |
           python3 -m pip install cmake
           python3 -m pip install -r requirements/build.txt
           # use cached build
-          cp -r ../../build build
+          mkdir build
           cd build
           cmake .. \
               -DCMAKE_BUILD_TYPE=RelWithDebInfo \
@@ -84,7 +85,9 @@ jobs:
           make -j$(nproc) && make install
       - name: Install lmdeploy
         run: |
-          python3 -m pip install packaging protobuf transformers_stream_generator
+          # manually install flash attn
+          python3 -m pip install /root/packages/flash_attn-2.3.6+cu118torch2.0cxx11abiFALSE-cp38-cp38-linux_x86_64.whl
+          python3 -m pip install packaging protobuf transformers_stream_generator transformers==4.33.0
           python3 -m pip install -r requirements.txt -r requirements/test.txt
           python3 -m pip install .
       - name: Check env
@@ -93,17 +96,14 @@ jobs:
           lmdeploy check_env
       - name: Test lmdeploy
         run: |
-          echo "TODO: awaiting PR of adding autotest"
-          # pytest autotest ${{github.event.inputs.markers}} --alluredir=allure-results --clean-alluredir
+          pytest autotest ${{github.event.inputs.markers}} --alluredir=allure-results --clean-alluredir
       - name: Generate reports
         if: always()
         run: |
-          if test -D "allure-results"; then
-            export date_today="$(date +'%Y%m%d-%H%M%S')"
-            export report_dir="$REPORT_DIR/$date_today"
-            echo "Save report to $ALLURE_DIR"
-            allure generate -c -o $report_dir
-          fi
+          export date_today="$(date +'%Y%m%d-%H%M%S')"
+          export report_dir="$REPORT_DIR/$date_today"
+          echo "Save report to $ALLURE_DIR"
+          allure generate -c -o $report_dir
       - name: Clear workfile
         if: always()
         run: |

autotest/chat_prompt_case.yaml

@@ -0,0 +1,64 @@
+identity_case:
+    - 你好，你叫什么名字#hi, what's your name:
+
+memory_test:
+    - 介绍成都的景点#please introduce some attractions in Chengdu:
+        - contain:
+            - chengdu
+            - 成都
+        - contain:
+            - 熊猫
+            - panda
+    - 介绍美食#please introduce some delicious foods:
+        - contain:
+            - 成都
+            - chengdu
+        - contain:
+            - 火锅
+            - hotpot
+            - hot pot
+new_session_without_memory:
+    - 介绍成都的景点#please introduce some attractions in Chengdu:
+        - contain:
+            - chengdu
+            - 成都
+    - end:
+    - 介绍美食#please introduce some delicious foods:
+        - not_contain:
+            - 成都
+            - chengdu
+chinese_poem_case:
+    - 给我一首中文打油诗，需要添加标点符号。和，Give me a Chinese poem in Chinese:
+        - contain:
+            - "，"
+            - "。"
+            - 油
+        - len_g:
+            10
+english_poem_case:
+    - write a romantic poem:
+        - contain:
+            - ","
+        - contain:
+            - "."
+        - contain:
+            - love
+        - len_g:
+            100
+emoji_case:
+    - 你能输出👍的emoji么#Can you output the emoji of 👍:
+        - contain:
+            - 👍
+            - 😊
+traditional_chinese_case:
+    - 請介紹香港維多利亞港，用繁體中文回答:
+        - contain:
+            - 維多利亞港
+            - 维多利亚港
+            - victoria
+session_len_error:
+    - 这是很长的一段话这是很长的一段话这是很长的一段话这是很长的一段话这是很长的一段话:
+        - contain:
+            - exceed session max length.
+        - contain:
+            - Please end the session.

autotest/config.yaml

@@ -0,0 +1,23 @@
+model_path: /nvme/qa_test_models
+dst_path: /nvme/qa_test_models/autotest_model
+log_path: /nvme/qa_test_models/autotest_model/log
+gradio_log_path: /lmdeploy/autotest/gradiotest
+
+
+model_map:
+    llama2-chat-7b-w4: llama2
+    internlm-chat-7b: internlm-chat-7b
+    internlm-chat-20b: internlm-chat-20b
+    internlm-chat-20b-inner-w4: internlm-chat-20b
+    Qwen-7B-Chat: qwen-7b
+    Qwen-7B-Chat-inner-w4: qwen-7b
+    Qwen-14B-Chat: qwen-14b
+    Qwen-14B-Chat-inner-w4: qwen-14b
+    Baichuan2-7B-Chat: baichuan2-7b
+    Baichuan2-7B-Chat-inner-w4: baichuan2-7b
+    CodeLlama-7b-Instruct-hf: codellama
+    CodeLlama-7b-Instruct-hf-inner-w4: codellama
+    Llama-2-7b-chat-hf: llama2
+    Llama-2-7b-chat-hf-inner-w4: llama2
+    llama-2-7b-chat: llama2
+    llama-2-7b-chat-inner-w4: llama2

autotest/conftest.py

@@ -0,0 +1,42 @@
+import os
+
+import pytest
+import yaml
+
+
+@pytest.fixture(scope='session')
+def config(request):
+    config_path = os.path.join(request.config.rootdir, 'config.yaml')
+    print(config_path)
+    with open(config_path) as f:
+        env_config = yaml.load(f.read(), Loader=yaml.SafeLoader)
+    return env_config
+
+
+@pytest.fixture(scope='session')
+def case_config(request):
+    case_path = os.path.join(request.config.rootdir, 'chat_prompt_case.yaml')
+    print(case_path)
+    with open(case_path) as f:
+        case_config = yaml.load(f.read(), Loader=yaml.SafeLoader)
+    return case_config
+
+
+@pytest.fixture(scope='class', autouse=True)
+def restful_case_config(request):
+    case_path = os.path.join(request.config.rootdir,
+                             'restful_prompt_case.yaml')
+    print(case_path)
+    with open(case_path) as f:
+        case_config = yaml.load(f.read(), Loader=yaml.SafeLoader)
+        del case_config['session_len_error']
+    return case_config
+
+
+@pytest.fixture(scope='session')
+def get_restful_case_List(request, case_config):
+    case_path = os.path.join(request.config.rootdir, 'chat_prompt_case.yaml')
+    print(case_path)
+    with open(case_path) as f:
+        case_config = yaml.load(f.read(), Loader=yaml.SafeLoader)
+    return case_config

autotest/pytest.ini

@@ -0,0 +1,8 @@
+[pytest]
+; addopts = --html=../report/report.html # pytest-html报告插件配置
+;addopts = -s --alluredir report      # allure-pytest报告插件配置
+; testpaths = ./scripts    # 设置测试目录识别名称
+python_files = test*_*.py  # 设置测试文件识别名称
+python_classes = Test*     # 设置测试类识别名称
+python_functions = test_*  # 设置测试方法识别名称
+filterwarnings = ignore::UserWarning

autotest/restful_prompt_case.yaml

@@ -0,0 +1,59 @@
+identity_case:
+    - 你好，你叫什么名字#hi, what's your name:
+
+memory_test:
+    - 介绍成都的景点#please introduce some attractions in Chengdu:
+        - contain:
+            - chengdu
+            - 成都
+        - contain:
+            - 熊猫
+            - panda
+    - 介绍美食#please introduce some delicious foods:
+        - contain:
+            - 成都
+            - chengdu
+        - contain:
+            - 火锅
+            - hotpot
+            - hot pot
+new_session_without_memory:
+    - 介绍美食#please introduce some delicious foods:
+        - not_contain:
+            - 成都
+            - chengdu
+chinese_poem_case:
+    - 给我一首中文打油诗，需要添加标点符号。和，Give me a Chinese poem in Chinese:
+        - contain:
+            - "，"
+            - "。"
+            - 油
+        - len_g:
+            10
+english_poem_case:
+    - write a romantic poem:
+        - contain:
+            - ","
+        - contain:
+            - "."
+        - contain:
+            - love
+        - len_g:
+            100
+emoji_case:
+    - 你能输出👍的emoji么#Can you output the emoji of 👍:
+        - contain:
+            - 👍
+            - 😊
+traditional_chinese_case:
+    - 請介紹香港維多利亞港，用繁體中文回答:
+        - contain:
+            - 維多利亞港
+            - 维多利亚港
+            - victoria
+session_len_error:
+    - 这是很长的一段话这是很长的一段话这是很长的一段话这是很长的一段话这是很长的一段话:
+        - contain:
+            - exceed session max length.
+        - contain:
+            - Please end the session.

autotest/test_order1_w4_quantization.py

@@ -0,0 +1,125 @@
+import os
+import subprocess
+from subprocess import PIPE
+
+import allure
+import pytest
+
+
+@pytest.mark.quantization
+@pytest.mark.CodeLlama_7b_Instruct_hf
+@allure.story('CodeLlama-7b-Instruct-hf')
+def future_test_quantization_CodeLlama_7b_Instruct_hf(config):
+    result = quantization(config, 'CodeLlama-7b-Instruct-hf-inner-w4',
+                          'CodeLlama-7b-Instruct-hf')
+    assert result.get('success'), result.get('msg')
+
+
+@pytest.mark.quantization
+@pytest.mark.llama_2_7b_chat
+@allure.story('llama-2-7b-chat')
+def test_quantization_llama_2_7b_chat(config):
+    result = quantization(config, 'llama-2-7b-chat-inner-w4',
+                          'llama-2-7b-chat')
+    assert result.get('success'), result.get('msg')
+
+
+@pytest.mark.quantization
+@pytest.mark.internlm_chat_20b
+@allure.story('internlm-chat-20b')
+def test_quantization_internlm_chat_20b(config):
+    result = quantization(config, 'internlm-chat-20b-inner-w4',
+                          'internlm-chat-20b')
+    assert result.get('success'), result.get('msg')
+
+
+@pytest.mark.quantization
+@pytest.mark.Qwen_14B_Chat
+@allure.story('Qwen-14B-Chat')
+def test_quantization_Qwen_14B_Chat(config):
+    result = quantization(config, 'Qwen-14B-Chat-inner-w4', 'Qwen-14B-Chat')
+    assert result.get('success'), result.get('msg')
+
+
+@pytest.mark.quantization
+@pytest.mark.Baichuan2_7B_Chat
+@allure.story('Baichuan2-7B-Chat')
+def test_quantization_Baichuan2_7B_Chat(config):
+    result = quantization(config, 'Baichuan2-7B-Chat-inner-w4',
+                          'Baichuan2-7B-Chat')
+    assert result.get('success'), result.get('msg')
+
+
+@pytest.mark.quantization
+@pytest.mark.Qwen_7B_Chat
+@allure.story('Qwen-7B-Chat')
+def test_quantization_Qwen_7B_Chat(config):
+    result = quantization(config, 'Qwen-7B-Chat-inner-w4', 'Qwen-7B-Chat')
+    assert result.get('success'), result.get('msg')
+
+
+def quantization(config, w4_model_name, origin_model_name):
+    model_path = config.get('model_path')
+    log_path = config.get('log_path')
+
+    get_param_cmd = [
+        'lmdeploy lite calibrate --model ' + model_path + '/' +
+        origin_model_name + ' --calib_dataset \'c4\' --calib_samples 128' +
+        ' --calib_seqlen 2048 --work_dir ' + model_path + '/' + w4_model_name
+    ]
+    quantization_cmd = [
+        'lmdeploy lite auto_awq --model ' + model_path + '/' +
+        origin_model_name + ' --w_bits 4  --w_group_size 128  ' +
+        '--work_dir ' + model_path + '/' + w4_model_name
+    ]
+    quantization_param_log = os.path.join(
+        log_path, 'quantization_param_' + w4_model_name + '.log')
+    quantization_log = os.path.join(log_path,
+                                    'quantization_' + w4_model_name + '.log')
+
+    with allure.step('step1 - get quantization params'):
+        with open(quantization_param_log, 'w') as f:
+            subprocess.run(['pwd'],
+                           stdout=f,
+                           stderr=f,
+                           shell=True,
+                           text=True,
+                           encoding='utf-8')
+            f.writelines('commondLine get_param_cmd: ' +
+                         ' '.join(get_param_cmd) + '\n')
+            # get params
+            getParamsRes = subprocess.run(get_param_cmd,
+                                          stdout=f,
+                                          stderr=PIPE,
+                                          shell=True,
+                                          text=True,
+                                          encoding='utf-8')
+            f.writelines(getParamsRes.stderr)
+            result = getParamsRes.returncode == 0
+
+        allure.attach.file(quantization_param_log,
+                           attachment_type=allure.attachment_type.TEXT)
+        if result is False:
+            return {
+                'success': result,
+                'quantization_param_log': quantization_param_log,
+                'msg': getParamsRes.stderr
+            }
+
+    with allure.step('step2 - quantization'):
+        with open(quantization_log, 'w') as f:
+            f.writelines('commondLine quantization_cmd: ' +
+                         ' '.join(quantization_cmd) + '\n')
+            # convert
+            quantizationRes = subprocess.run(quantization_cmd,
+                                             stdout=f,
+                                             stderr=PIPE,
+                                             shell=True,
+                                             text=True,
+                                             encoding='utf-8')
+            f.writelines(quantizationRes.stderr)
+            result = quantizationRes.returncode == 0
+
+    allure.attach.file(quantization_log,
+                       attachment_type=allure.attachment_type.TEXT)
+    return {'success': result, 'msg': getParamsRes.stderr}