[Feature] Update Models, Summarizers (#1600)

opencompass/configs/models/hf_llama/hf_llama3_2_3b_instruct.py

@@ -0,0 +1,13 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='llama-3_2-3b-instruct-hf',
+        path='meta-llama/Llama-3.2-3B-Instruct',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
+        stop_words=['<|end_of_text|>', '<|eot_id|>'],
+    )
+]

opencompass/configs/models/hf_llama/lmdeploy_llama3_2_3b_instruct.py

@@ -0,0 +1,16 @@
+from opencompass.models import TurboMindModelwithChatTemplate
+
+models = [
+    dict(
+        type=TurboMindModelwithChatTemplate,
+        abbr='llama-3_2-3b-instruct-turbomind',
+        path='meta-llama/Llama-3.2-3B-Instruct',
+        engine_config=dict(max_batch_size=16, tp=1),
+        gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=4096),
+        max_seq_len=16384,
+        max_out_len=4096,
+        batch_size=16,
+        run_cfg=dict(num_gpus=1),
+        stop_words=['<|end_of_text|>', '<|eot_id|>'],
+    )
+]

opencompass/configs/models/mistral/hf_mistral_nemo_instruct_2407.py

@@ -0,0 +1,12 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='mistral-nemo-instruct-2407-hf',
+        path='mistralai/Mistral-Nemo-Instruct-2407',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
+    )
+]

opencompass/configs/models/mistral/hf_mistral_small_instruct_2409.py

@@ -0,0 +1,12 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='mistral-small-instruct-2409-hf',
+        path='mistralai/Mistral-Small-Instruct-2409',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=2),
+    )
+]

opencompass/configs/models/mistral/lmdeploy_mistral_nemo_instruct_2407.py

@@ -0,0 +1,15 @@
+from opencompass.models import TurboMindModelwithChatTemplate
+
+models = [
+    dict(
+        type=TurboMindModelwithChatTemplate,
+        abbr='mistral-nemo-instruct-2407-turbomind',
+        path='mistralai/Mistral-Nemo-Instruct-2407',
+        engine_config=dict(session_len=32768, max_batch_size=16, tp=1),
+        gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=4096),
+        max_seq_len=32768,
+        max_out_len=4096,
+        batch_size=16,
+        run_cfg=dict(num_gpus=1),
+    )
+]

opencompass/configs/models/mistral/lmdeploy_mistral_small_instruct_2409.py

@@ -0,0 +1,15 @@
+from opencompass.models import TurboMindModelwithChatTemplate
+
+models = [
+    dict(
+        type=TurboMindModelwithChatTemplate,
+        abbr="mistral-small-instruct-2409-turbomind",
+        path="mistralai/Mistral-Small-Instruct-2409",
+        engine_config=dict(session_len=32768, max_batch_size=16, tp=2),
+        gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=4096),
+        max_seq_len=32768,
+        max_out_len=4096,
+        batch_size=16,
+        run_cfg=dict(num_gpus=2),
+    )
+]

opencompass/datasets/compassbench_obj.py

@@ -26,7 +26,7 @@ def load(path: str, name: str):
         circular_patterns = ['ABCD', 'BCDA', 'CDAB', 'DABC']
 
         data = []
-        with open(path, 'r') as infile:
+        with open(path, 'r', encoding='utf-8', errors='ignore') as infile:
             for id, line in enumerate(infile):
                 entry = json.loads(line)
                 if 'cloze' in name:

opencompass/models/bailing_api_oc.py

@@ -81,8 +81,8 @@ def __init__(
             self._headers = {'Authorization': f'Bearer {token}'}
 
         self._headers['Content-Type'] = 'application/json'
-        self._url = url if url else \
-            'https://bailingchat.alipay.com/chat/completions'
+        self._url = (url if url else
+                     'https://bailingchat.alipay.com/chat/completions')
         self._model = path
         self._sessions = []
         self._num = (int(os.environ.get('BAILING_API_PARALLEL_NUM'))
@@ -136,9 +136,9 @@ def generate(
                         results.append('')
                     else:
                         if (result.get('choices')
-                                and result['choices'][0].get('message')
-                                and result['choices'][0]['message'].get(
-                                    'content')):
+                                and result['choices'][0].get('message') and
+                                result['choices'][0]['message'].get('content')
+                                is not None):
                             results.append(
                                 result['choices'][0]['message']['content'])
                 else:

opencompass/models/openai_api.py

@@ -466,25 +466,28 @@ def bin_trim(self, prompt: str, num_token: int) -> str:
 
 class OpenAISDK(OpenAI):
 
-    def __init__(self,
-                 path: str = 'gpt-3.5-turbo',
-                 max_seq_len: int = 4096,
-                 query_per_second: int = 1,
-                 rpm_verbose: bool = False,
-                 retry: int = 2,
-                 key: str | List[str] = 'ENV',
-                 org: str | List[str] | None = None,
-                 meta_template: Dict | None = None,
-                 openai_api_base: str = OPENAI_API_BASE,
-                 openai_proxy_url: Optional[str] = None,
-                 mode: str = 'none',
-                 logprobs: bool | None = False,
-                 top_logprobs: int | None = None,
-                 temperature: float | None = None,
-                 tokenizer_path: str | None = None,
-                 extra_body: Dict | None = None,
-                 max_completion_tokens: int = 16384,
-                 verbose: bool = False):
+    def __init__(
+        self,
+        path: str = 'gpt-3.5-turbo',
+        max_seq_len: int = 4096,
+        query_per_second: int = 1,
+        rpm_verbose: bool = False,
+        retry: int = 2,
+        key: str | List[str] = 'ENV',
+        org: str | List[str] | None = None,
+        meta_template: Dict | None = None,
+        openai_api_base: str = OPENAI_API_BASE,
+        openai_proxy_url: Optional[str] = None,
+        mode: str = 'none',
+        logprobs: bool | None = False,
+        top_logprobs: int | None = None,
+        temperature: float | None = None,
+        tokenizer_path: str | None = None,
+        extra_body: Dict | None = None,
+        max_completion_tokens: int = 16384,
+        verbose: bool = False,
+        status_code_mappings: dict = {},
+    ):
         super().__init__(path,
                          max_seq_len,
                          query_per_second,
@@ -519,9 +522,11 @@ def __init__(self,
                 http_client=httpx.Client(proxies=proxies))
         if self.verbose:
             self.logger.info(f'Used openai_client: {self.openai_client}')
+        self.status_code_mappings = status_code_mappings
 
     def _generate(self, input: PromptList | str, max_out_len: int,
                   temperature: float) -> str:
+        from openai import BadRequestError
         assert isinstance(input, (str, PromptList))
 
         # max num token for gpt-3.5-turbo is 4097
@@ -605,7 +610,30 @@ def _generate(self, input: PromptList | str, max_out_len: int,
                         self.logger.info(responses)
                     except Exception as e:  # noqa F841
                         pass
+                if not responses.choices:
+                    self.logger.error(
+                        'Response is empty, it is an internal server error \
+                            from the API provider.')
                 return responses.choices[0].message.content
+
+            except BadRequestError as e:
+                # Handle BadRequest status
+                # You can specify self.status_code_mappings to bypass \
+                # API sensitivity blocks
+                # For example: status_code_mappings={400: 'Input data \
+                # may contain inappropriate content.'}
+                status_code = e.status_code
+                if (status_code is not None
+                        and status_code in self.status_code_mappings):
+                    original_error_message = e.body.get('message')
+                    error_message = self.status_code_mappings[status_code]
+                    self.logger.info(
+                        f'Status Code: {status_code}, '
+                        f'Original Error Message: {original_error_message},'
+                        f'Return Message: {error_message} ')
+                    return error_message
+                else:
+                    self.logger.error(e)
             except Exception as e:
                 self.logger.error(e)
             num_retries += 1