新老用户看过来～最实用的 Milvus 迁移手册来啦！

dumper:  worker:    workMode: elasticsearch        ------ 工作模式：elasticsearch    reader:      bufferSize: 2500             --------- 从es每次批量获取的数量meta:  mode: config                     -------- 固定写法，其他数据源迁移会有其他不同值    index: test_index                -------- es index  fields:                          -------- 需要同步的es字段有哪些：        - name: id                     -------- es 字段名      pk: true                     -------- pk=true,表示这个字段作为milvus的主键, 没设置情况下默认会采用es document _id作为主键                type: long    - name: other_field      maxLen: 60                   ------- 对应milvus VarChar字段类型的maxLen, 对于varchar类型不设置maxLen则默认最大值：65535      type: keyword    - name: data      type: dense_vector           ------- 向量字段，对应milvus的 field_vector类型, 必须迁移有dense_vector的字段      dims: 512  milvus:                          ------- 这部分配置非必填，设置生成的milvus表的属性，为空则按默认值      collection: "rename_index_test"  --- 表名，为空则 esIndex作为表名      closeDynamicField: false         --- 为空默认为false(开启动态列功能)      consistencyLevel: Eventually   --- 一致性，为空按 milvus的默认级别      shardNum: 1                      --- 分片数量,为空默认为2
source:                        ------- es server连接配置信息,支持 serviceToken,fingerprint,cloudId/apiKey,user/pwd,ca.crt 等方式连接/认证  es:    urls:      - http://localhost:9200    username: xxx    password: xxxtarget:                       ------ 迁移到目标mivlus的bucket信息  mode: remote  remote:    outputDir: outputPath/migration/test1    cloud: aws    region: us-west-2    bucket: xxx    useIAM: true    checkBucket: false  milvus2x:    endpoint: {yourMilvusAddress}:{port}    username: ******    password: ******

[task/load_base_task.go:94] ["[LoadTasker] Dec Task Processing-------------->"] [Count=0] [fileName=testfiles/output/zwh/migration/test_mul_field4/data_1_1.json] [taskId=442665677354739304][task/load_base_task.go:76] ["[LoadTasker] Progress Task --------------->"] [fileName=testfiles/output/zwh/migration/test_mul_field4/data_1_1.json] [taskId=442665677354739304][dbclient/cus_field_milvus2x.go:86] ["[Milvus2x] begin to ShowCollectionRows"][loader/cus_milvus2x_loader.go:66] ["[Loader] Static: "] [collection=test_mul_field4_rename1] [beforeCount=50000] [afterCount=100000] [increase=50000][loader/cus_milvus2x_loader.go:66] ["[Loader] Static Total"] ["Total Collections"=1] [beforeTotalCount=50000] [afterTotalCount=100000] [totalIncrease=50000][migration/es_starter.go:25] ["[Starter] migration ES to Milvus finish!!!"] [Cost=80.009174459][starter/starter.go:106] ["[Starter] Migration Success!"] [Cost=80.00928425][cleaner/remote_cleaner.go:27] ["[Remote Cleaner] Begin to clean files"] [bucket=a-bucket] [rootPath=testfiles/output/zwh/migration][cmd/start.go:32] ["[Cleaner] clean file success!"]

--- meta.json    
--- tables

dumper:  worker:    limit: 2    workMode: milvus1x    ------ 工作模式：milvus1x    reader:      bufferSize: 1024      ----- file reader/writer buffer size    writer:      bufferSize: 1024loader:  worker:    limit: 16       ------- 支持同时并发迁移的表数量meta:  mode: local    ------ meta.json文件存放方式，有：local, remote, mysql, sqlite,   localFile: /outputDir/test/meta.json  -- mode: mysql  # milvus的元数据mysql地址  -- mysqlUrl: "user:password@tcp(localhost:3306)/milvus?charset=utf8mb4&parseTime=True&loc=Local"source:        ----- milvus1.x talbes目录文件存储源，可以在local, s3,minio,oss,gcp  mode: local  local:    tablesDir: /db/tables/  --数据文件tables目录target:           ------ 数据通过bulkInsert写入的目标存储位置，  mode: remote    --- 写入的存储的方式可以可以是：remote 和 local(仅验证dump功能使用)  remote:    outputDir: "migration/test/xx" --写入的路径    ak: xxxx    sk: xxxx    cloud: aws   ------ 写入的cloud, 可以是aws, gcp, ali， (如果是minio也填写aws)    region: us-west-2    bucket: xxxxx    useIAM: true    checkBucket: false  milvus2x:    endpoint: "{yourMilvus2_xServerAddress}:{port}"    username: xxxx    password: xxxx

dumper:  worker:    limit: 2    workMode: faiss    ------ 工作模式：faiss    reader:      bufferSize: 1024    writer:      bufferSize: 1024loader:  worker:    limit: 2source:  mode: local    ---- 数据源可以为 local和 remote  local:    faissFile: ./testfiles/faiss/faiss_ivf_flat.index
target:  create:            ----- 指定生成的collection属性    collection:      name: test1w      shardsNums: 2      dim: 256      metricType: L2
  mode: remote  remote:            ------- 下面配置是将数据写入到本地搭建的minio中，milvus2x也是本地    outputDir: testfiles/output/    cloud: aws       ---- 同样支持 aws, gcp, ali     endpoint: 0.0.0.0:9000    region: ap-southeast-1    bucket: a-bucket    ak: minioadmin            sk: minioadmin    useIAM: false    useSSL: false    checkBucket: true  milvus2x:    endpoint: localhost:19530    username: xxxxx    password: xxxxx