如何处理海量数据？基于 Milvus 向量数据库的高度可扩展性

from pymilvus import MilvusClient
client = MilvusClient(uri=<ZILLIZ_CLOUD_URI>, token=<ZILLIZ_TOKEN>)

# Define the schema for the collectionembedding_dim = 1024
schema = [            {"name": "id", "dtype": "int64", "is_primary": True, "auto_id": True},            {"name": "text_vector", "dtype": "float_vector", "dim": embedding_dim},            {"name": "title", "dtype": "varchar", "max_length": 5000},            {"name": "text", "dtype": "varchar", "max_length": 5000},        ]

# Create the collection if it doesn't existcollection_name = "cohere_embeddings"
if not client.has_collection(collection_name):    client.create_collection(collection_name=collection_name, schema=schema)

# Define and create indexindex_params = {    "metric_type": "COSINE",    "index_type": "HNSW",    "params": {"M": 8, "efConstruction": 64},}client.create_index(collection_name=collection_name, field_name="text_vector", index_params=index_params)
# Load the collectionclient.load_collection(collection_name=collection_name)

# Insert data into our collectiondef insert_batch(client, collection_name, batch_data):    client.insert(collection_name=collection_name, data=batch_data)    batch_data.clear()
# Get the data from HuggingFace and create some batchdef insert_data(client, collection_name):    batch_size = 1000  # Adjust the batch size as needed    batch_data = []
    docs = load_dataset(        "Cohere/wikipedia-2023-11-embed-multilingual-v3",        "en", # English only        split="train",        streaming=True, # Allows us to iterate over the dataset    )
    for doc in tqdm(docs, desc="Streaming and preparing data for Milvus"):        title = doc["title"][:4500] # Titles can be very long        text = doc["text"][:4500] # Text can be very long        emb = doc["emb"]  # The embedding vector
        batch_data.append({"title": title, "text_vector": emb, "text": text})        if len(batch_data) >= batch_size:            insert_batch(client, collection_name, batch_data)    if batch_data:        insert_batch(client, collection_name, batch_data)

FILTER_TITLE = "British Arab Commercial Bank"
res = milvus_client.query(    collection_name=collection_name,    filter=f'title like "{FILTER_TITLE}"',    output_fields=["title", "text"])for elt in res:     pprint(elt)

{'id': 450933285225527270, 'text': 'The British Arab Commercial Bank PLC (BACB) is an international '         'wholesale bank incorporated in the United Kingdom that is authorised '         'by the Prudential Regulation Authority (PRA) and regulated by the '         'PRA and the Financial Conduct Authority (FCA). It was founded in '         '1972 as UBAF Limited, adopted its current name in 1996, and '         'registered as a public limited company in 2009. The bank has clients '         'trading in and out of developing markets in the Middle East and '         'Africa.', 'title': 'British Arab Commercial Bank'}{'id': 450933285225527271, 'text': 'BACB has a head office in London, and three representative offices '         'in Algiers in Algeria, Tripoli in Libya and Abidjan in the Cote '         "D'Ivoire. The bank has 17 sister banks across Europe, Asia and "         'Africa. It is owned by three main shareholders - the Libyan Foreign '         'Bank (87.80%), Banque Centrale Populaire (6.10%) and Banque '         "Extérieure d'Algérie (6.10%).", 'title': 'British Arab Commercial Bank'}

res = milvus_client.query(    collection_name=collection_name,    filter='text like "%Calectasia%"', # Infix    # filter='text like "Calect%"', # Suffix    # filter='text like "%lectasia"', # Prefix    output_fields=["title", "text"],    limit=5,)for elt in res:     pprint(elt)

{'id': 450933285225527360, 'text': 'Calectasia is a genus of about fifteen species of flowering plants '         'in the family Dasypogonaceae and is endemic to south-western '         'Australia. Plants is this genus are small, erect shrubs with '         'branched stems covered by leaf sheaths. The flowers are star-shaped, '         'lilac-blue to purple and arranged singly on the ends of short '         'branchlets.', 'title': 'Calectasia'}{'id': 450933285225527361, 'text': 'Plants in the genus Calectasia are small, often rhizome-forming '         'shrubs with erect, branched stems with sessile leaves arranged '         'alternately along the stems,  long and about  wide, the base held '         'closely against the stem and the tip pointed. The flowers are '         'arranged singly on the ends of branchlets and are bisexual, the '         'three sepals and three petals are similar to each other, and joined '         'at the base forming a short tube but spreading, forming a star-like '         'pattern with a metallic sheen. Six bright yellow or orange stamens '         'form a tube in the centre of the flower with a thin style extending '         'beyond the centre of the tube.', 'title': 'Calectasia'}

res = milvus_client.query(    collection_name=collection_name,    filter='title not in ["British Arab Commercial Bank", "Calectasia"]',    output_fields=["title", "text"],    limit=10,)for elt in res:     pprint(elt)

{'id': 450933285225527281, 'text': 'The Commonwealth Skyranger, first produced as the Rearwin Skyranger, '         'was the last design of Rearwin Aircraft before the company was '         'purchased by a new owner and renamed Commonwealth Aircraft.  It was '         'a side-by-side, two-seat, high-wing taildragger.', 'title': 'Commonwealth Skyranger'}{'id': 450933285225527282, 'text': 'The Rearwin company had specialized in aircraft powered by small '         'radial engines, such as their Sportster and Cloudster, and had even '         'purchased the assets of LeBlond Engines to make small radial engines '         'in-house in 1937. By 1940, however, it was clear Rearwin would need '         'a design powered by a small horizontally opposed engine to remain '         'competitive. Intended for sport pilots and flying businessmen, the '         '"Rearwin Model 165" first flew on April 9, 1940. Originally named '         'the "Ranger," Ranger Engines (who also sold several engines named '         '"Ranger") protested, and Rearwin renamed the design "Skyranger." The '         'overall design and construction methods allowed Rearwin to take '         'orders for Skyrangers then deliver the aircraft within 10 weeks.', 'title': 'Commonwealth Skyranger'}

from pymilvus.model.dense import CohereEmbeddingFunction
cohere_ef = CohereEmbeddingFunction(    model_name="embed-multilingual-v3.0",    input_type="search_query",    embedding_types=["float"])
query = 'Who founded Wikipedia'embedded_query = cohere_ef.encode_queries([query])
response = embedded_query[0]print(response[:10])

res = milvus_client.search(data=response, collection_name=collection_name, output_fields=["text"], limit=3)
for elt in res:    pprint(elt)

[{'distance': 0.7344469428062439,  'entity': {'text': 'Larry Sanger and Jimmy Wales are the ones who started '                     'Wikipedia. Wales is credited with defining the goals of '                     'the project. Sanger created the strategy of using a wiki '                     "to reach Wales' goal. On January 10, 2001, Larry Sanger "                     'proposed on the Nupedia mailing list to create a wiki as '                     'a "feeder" project for Nupedia. Wikipedia was launched '                     'on January 15, 2001. It was launched as an '                     'English-language edition at www.wikipedia.com, and '                     'announced by Sanger on the Nupedia mailing list. '                     'Wikipedia\'s policy of "neutral point-of-view" was '                     'enforced in its initial months and was similar to '                     'Nupedia\'s earlier "nonbiased" policy. Otherwise, there '                     "weren't very many rules initially, and Wikipedia "                     'operated independently of Nupedia.'},  'id': 450933285241797095}, {'distance': 0.7239157557487488,  'entity': {'text': 'Wikipedia was originally conceived as a complement to '                     'Nupedia, a free on-line encyclopedia founded by Jimmy '                     'Wales, with articles written by highly qualified '                     'contributors and evaluated by an elaborate peer review '                     'process. The writing of content for Nupedia proved to be '                     'extremely slow, with only 12 articles completed during '                     'the first year, despite a mailing-list of interested '                     'editors and the presence of a full-time editor-in-chief '                     'recruited by Wales, Larry Sanger. Learning of the wiki '                     'concept, Wales and Sanger decided to try creating a '                     'collaborative website to provide an additional source of '                     'rapidly produced draft articles that could be polished '                     'for use on Nupedia.'},  'id': 450933285225827849}, {'distance': 0.7191773653030396,  'entity': {'text': "The foundation's creation was officially announced by "                     'Wikipedia co-founder Jimmy Wales, who was running '                     'Wikipedia within his company Bomis, on June 20, 2003.'},  'id': 450933285242058780}]

> curl --request POST \     --url https://api.cloud.zilliz.com/v2/clusters/<cluster_id>/metrics/query \     [...]       "metricQueries": [           {               "name": "REQ_SEARCH_LATENCY",               "stat": "AVG"           }    }'    [{"name":"REQ_SEARCH_LATENCY","stat":"AVG","unit":"millisecond","values":[{"timestamp":"2024-08-26T11:09:53Z","value":"2.0541596873255163"}]}]

> curl --request POST \     --url https://api.cloud.zilliz.com/v2/clusters/<cluster_id>/metrics/query \     [...]       "metricQueries": [           {               "name": "REQ_SEARCH_LATENCY",               "stat": "P99"           }    }'[{"name":"REQ_SEARCH_LATENCY","stat":"P99","unit":"millisecond","values":[{"timestamp":"2024-08-26T11:09:53Z","value":"4.949999999999999"}]}]

context = "\n".join(    [line_with_distance[0] for line_with_distance in retrieved_lines_with_distances])
question = "Who created Wikipedia?"
SYSTEM_PROMPT = """Human: You are an AI assistant. You are able to find answers to the questions from the contextual passage snippets provided."""
USER_PROMPT = f"""    Use the following pieces of information enclosed in <context> tags to provide an answer to the question enclosed in <question> tags.    <context>    {context}    </context>    <question>    {question}    </question>"""

from openai import OpenAI
client = OpenAI(    base_url = 'http://localhost:11434/v1',    api_key='ollama', # required, but unused)
response = client.chat.completions.create(    model="llama3.1",    messages=[        {"role": "system", "content": SYSTEM_PROMPT},        {"role": "user", "content": USER_PROMPT},    ],)print(response.choices[0].message.content)

According to the provided context, Larry Sanger and Jimmy Wales are the ones who started Wikipedia. Specifically, Jimmy Wales defined the goals of the project, while Larry Sanger created the strategy of using a wiki to achieve those goals.