name: "chatglm2-6b"backend: "python"max_batch_size: 1
input [  {    name: "QUERY"    data_type: TYPE_STRING    dims: [ -1 ]  },  {    name: "max_new_tokens"    data_type: TYPE_UINT32    dims: [ -1 ]  },  {    name: "top_k"    data_type: TYPE_UINT32    dims: [ 1 ]    optional: true  },  {    name: "top_p"    data_type: TYPE_FP32    dims: [ 1 ]    optional: true  },  {    name: "temperature"    data_type: TYPE_FP32    dims: [ 1 ]    optional: true  },  {    name: "length_penalty"    data_type: TYPE_FP32    dims: [ 1 ]    optional: true  },  {    name: "repetition_penalty"    data_type: TYPE_FP32    dims: [ 1 ]    optional: true  },  {    name: "bos_token_id"    data_type: TYPE_UINT32    dims: [ 1 ]    optional: true  },  {    name: "eos_token_id"    data_type: TYPE_UINT32    dims: [ 1 ]    optional: true  },  {    name: "do_sample"    data_type: TYPE_BOOL    dims: [ 1 ]    optional: true  },  {    name: "num_beams"    data_type: TYPE_UINT32    dims: [ 1 ]    optional: true  }]output [  {    name: "OUTPUT"    data_type: TYPE_STRING    dims: [ -1, -1 ]  }]
instance_group [  {    kind: KIND_GPU  }]
parameters {  key: "model_path"  value: {    string_value: "/opt/tritonserver/python_backend/models/chatglm2-6b"  }}

    import triton_python_backend_utils as pb_utils

class TritonPythonModel:    @staticmethod    def auto_complete_config(auto_complete_model_config):        """`auto_complete_config` is called only once when loading the model
    def initialize(self, args):        """`initialize` is called only once when the model is being loaded.        Implementing `initialize` function is optional. This function allows        the model to initialize any state associated with this model.
        Parameters        ----------        args : dict          Both keys and values are strings. The dictionary keys and values are:          * model_config: A JSON string containing the model configuration          * model_instance_kind: A string containing model instance kind          * model_instance_device_id: A string containing model instance device            ID          * model_repository: Model repository path          * model_version: Model version          * model_name: Model name        """        print('Initialized...')
    def execute(self, requests):        """`execute` must be implemented in every Python model. `execute`        function receives a list of pb_utils.InferenceRequest as the only        argument. This function is called when an inference is requested        for this model.
        Parameters        ----------        requests : list          A list of pb_utils.InferenceRequest
        Returns        -------        list          A list of pb_utils.InferenceResponse. The length of this list must          be the same as `requests`        """
        responses = []
    def finalize(self):        """`finalize` is called only once when the model is being unloaded.        Implementing `finalize` function is optional. This function allows        the model to perform any necessary clean ups before exit.        """        print('Cleaning up...')

import requests# 定义模型的输入数据data = {    "inputs": [        {
            "name": "QUERY",            "shape": [1,1],            "datatype": "BYTES",            "data": ["川普是不是四川人"]        },        {
            "name": "max_new_tokens",            "shape" : [1,1],            "datatype": "UINT32",            "data": [15000]        },    ]}headers = {    'Content-Type': 'application/json',}# 发送 POST 请求response = requests.post('http://localhost:8000/v2/models/chatglm2-6b/infer', headers=headers, json=data)result = response.json()print(result)

{  "model_name": "chatglm2-6b",  "model_version": "1",  "outputs": [    {      "data": [        "\n\n 川普不是四川人，他出生于美国宾夕法尼亚州，是一个美国政治家、企业家和电视名人。"      ],      "datatype": "BYTES",      "name": "OUTPUT",      "shape": []    }  ]}